From 180c8bfb2c84abb65b0a6e6ba61510cb46e14c6f Mon Sep 17 00:00:00 2001 From: Eric Litman Date: Tue, 27 Oct 2026 11:31:34 -0400 Subject: [PATCH] HOUSE-79 part 07: upstream v8.0.44 changes for review --- internal/api/middleware/request_logging.go | 20 +- internal/api/middleware/response_writer.go | 6 + internal/api/protocol_multiplexer.go | 21 +- internal/api/redis_queue_protocol.go | 85 +++- internal/api/response_compression.go | 363 ++++++++++++++++++ internal/api/server.go | 60 +-- internal/api/server_management.go | 10 +- internal/api/server_management_v8.go | 12 +- internal/api/server_middleware.go | 103 ++++- internal/api/server_reload.go | 26 +- internal/api/server_routes.go | 203 ++++++++-- sdk/api/handlers/claude/code_handlers.go | 269 +++++++++++-- sdk/api/handlers/gemini/gemini_handlers.go | 75 +++- .../handlers/gemini/interactions_handlers.go | 65 +++- sdk/api/handlers/handlers.go | 124 +++++- sdk/api/handlers/handlers_errors.go | 4 +- sdk/api/handlers/handlers_execution.go | 29 +- sdk/api/handlers/handlers_interceptors.go | 58 ++- sdk/api/handlers/handlers_routing.go | 54 ++- sdk/api/handlers/handlers_stream.go | 79 ++-- sdk/api/handlers/model_execution.go | 12 + .../handlers/native_tool_search_capability.go | 10 + .../handlers/openai/codex_client_models.go | 18 +- sdk/api/handlers/openai/openai_handlers.go | 183 ++++++++- .../handlers/openai/openai_images_handlers.go | 210 ++++++---- .../openai/openai_responses_handlers.go | 210 ++++++++-- .../openai/openai_responses_websocket.go | 181 +++++++-- .../openai_responses_websocket_forward.go | 124 ++++-- .../openai_responses_websocket_input.go | 163 +++++++- .../openai_responses_websocket_timeline.go | 16 +- .../handlers/openai/openai_speech_handlers.go | 250 ++++++++++++ .../handlers/openai/openai_videos_handlers.go | 31 +- .../handlers/openai_responses_stream_error.go | 15 +- sdk/api/handlers/request_body.go | 31 +- sdk/api/handlers/stream_forwarder.go | 107 +++++- 35 files changed, 2789 insertions(+), 438 deletions(-) create mode 100644 internal/api/response_compression.go create mode 100644 sdk/api/handlers/native_tool_search_capability.go create mode 100644 sdk/api/handlers/openai/openai_speech_handlers.go diff --git a/internal/api/middleware/request_logging.go b/internal/api/middleware/request_logging.go index f5120a9b3..48f2d58ad 100644 --- a/internal/api/middleware/request_logging.go +++ b/internal/api/middleware/request_logging.go @@ -5,8 +5,10 @@ package middleware import ( "bytes" + "errors" "fmt" "io" + "math" "net/http" "os" "strings" @@ -52,6 +54,11 @@ func RequestLoggingMiddleware(logger logging.RequestLogger) gin.HandlerFunc { // Capture request information requestInfo, err := captureRequestInfo(c, captureBody) if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + c.AbortWithStatusJSON(http.StatusRequestEntityTooLarge, gin.H{"error": err.Error()}) + return + } // Log error but continue processing // In a real implementation, you might want to use a proper logger here c.Next() @@ -338,7 +345,12 @@ func captureRequestInfo(c *gin.Context, captureBody bool) (*RequestInfo, error) // Restore the body for the actual request processing c.Request.Body = io.NopCloser(bytes.NewBuffer(bodyBytes)) - body = decodeCapturedRequestBodyForLog(bodyBytes, c.Request.Header.Get("Content-Encoding")) + maxBytes := c.GetInt64("maxRequestBodyBytes") + if maxBytes > 0 { + body = decodeCapturedRequestBodyForLogWithLimit(bodyBytes, c.Request.Header.Get("Content-Encoding"), maxBytes) + } else { + body = decodeCapturedRequestBodyForLog(bodyBytes, c.Request.Header.Get("Content-Encoding")) + } } return &RequestInfo{ @@ -445,7 +457,11 @@ func decodeCapturedZstdRequestBodyWithLimit(raw []byte, limit int64) ([]byte, bo } defer decoder.Close() - decoded, errRead := io.ReadAll(io.LimitReader(decoder, limit+1)) + var reader io.Reader = decoder + if limit > 0 && limit < math.MaxInt64 { + reader = io.LimitReader(decoder, limit+1) + } + decoded, errRead := io.ReadAll(reader) if errRead != nil { return nil, false, fmt.Errorf("failed to decode zstd request body: %w", errRead) } diff --git a/internal/api/middleware/response_writer.go b/internal/api/middleware/response_writer.go index 44e1f0b66..0fe5b72be 100644 --- a/internal/api/middleware/response_writer.go +++ b/internal/api/middleware/response_writer.go @@ -13,6 +13,7 @@ import ( "github.com/gin-gonic/gin" "github.com/router-for-me/CLIProxyAPI/v8/internal/clienterror" + "github.com/router-for-me/CLIProxyAPI/v8/internal/httpwire" "github.com/router-for-me/CLIProxyAPI/v8/internal/interfaces" "github.com/router-for-me/CLIProxyAPI/v8/internal/logging" log "github.com/sirupsen/logrus" @@ -104,6 +105,11 @@ func (w *ResponseWriterWrapper) Write(data []byte) (int, error) { return n, err } +// FlushError forwards transport errors without bypassing inner middleware. +func (w *ResponseWriterWrapper) FlushError() error { + return httpwire.FlushResponse(w.ResponseWriter) +} + func (w *ResponseWriterWrapper) shouldBufferResponseBody() bool { if w.logger != nil && w.logger.IsEnabled() { return true diff --git a/internal/api/protocol_multiplexer.go b/internal/api/protocol_multiplexer.go index 3bcb578a2..ac9c62975 100644 --- a/internal/api/protocol_multiplexer.go +++ b/internal/api/protocol_multiplexer.go @@ -60,6 +60,21 @@ func (s *Server) acceptMuxConnections(listener net.Listener, httpListener *muxLi // routeMuxConnection performs per-connection protocol detection and routing. func (s *Server) routeMuxConnection(conn net.Conn, httpListener *muxListener) { + defer func() { + if recovered := recover(); recovered != nil { + var remoteAddr net.Addr + if conn != nil { + remoteAddr = conn.RemoteAddr() + } + log.Errorf("panic while routing connection from %v: %v", remoteAddr, recovered) + if conn != nil { + if errClose := conn.Close(); errClose != nil && !errors.Is(errClose, net.ErrClosed) { + log.Errorf("failed to close connection after panic: %v", errClose) + } + } + } + }() + // Set a read deadline so that idle connections that never send bytes do not // leak goroutines and file descriptors. The deadline is cleared once the // connection is successfully routed to its handler. @@ -82,12 +97,11 @@ func (s *Server) routeMuxConnection(conn net.Conn, httpListener *muxListener) { } return } + _ = conn.SetReadDeadline(time.Time{}) if errPut := httpListener.Put(tlsConn); errPut != nil { if errClose := conn.Close(); errClose != nil { log.Errorf("failed to close connection after HTTP routing failure: %v", errClose) } - } else { - _ = conn.SetReadDeadline(time.Time{}) } return } @@ -115,11 +129,10 @@ func (s *Server) routeMuxConnection(conn net.Conn, httpListener *muxListener) { return } + _ = conn.SetReadDeadline(time.Time{}) if errPut := httpListener.Put(&bufferedConn{Conn: conn, reader: reader}); errPut != nil { if errClose := conn.Close(); errClose != nil { log.Errorf("failed to close connection after HTTP routing failure: %v", errClose) } - } else { - _ = conn.SetReadDeadline(time.Time{}) } } diff --git a/internal/api/redis_queue_protocol.go b/internal/api/redis_queue_protocol.go index b70c4a4ea..2ca8b2598 100644 --- a/internal/api/redis_queue_protocol.go +++ b/internal/api/redis_queue_protocol.go @@ -17,6 +17,11 @@ import ( const ( redisUsageChannel = "usage" redisErrorsChannel = "errors" + + maxRESPArrayLen = 1024 + maxRESPBulkLen = 64 << 10 // 64 KiB + maxRESPFrameBytes = 64 << 10 // 64 KiB + maxRESPLineLen = 64 << 10 // 64 KiB ) type redisSubscriptionCommand struct { @@ -45,7 +50,10 @@ func (s *Server) handleRedisConnection(conn net.Conn, reader *bufio.Reader) { authed := false writer := bufio.NewWriter(conn) defer func() { - if errClose := conn.Close(); errClose != nil { + if recovered := recover(); recovered != nil { + log.Errorf("panic while handling redis connection from %v: %v", conn.RemoteAddr(), recovered) + } + if errClose := conn.Close(); errClose != nil && !errors.Is(errClose, net.ErrClosed) { log.Errorf("redis connection close error: %v", errClose) } }() @@ -58,7 +66,7 @@ func (s *Server) handleRedisConnection(conn net.Conn, reader *bufio.Reader) { return true } - if s.cfg != nil && s.cfg.Home.Enabled { + if cfg := s.getConfig(); cfg != nil && cfg.Home.Enabled { _ = writeRedisError(writer, "ERR redis usage output disabled in home mode") _ = writer.Flush() return @@ -289,6 +297,11 @@ func (s *Server) streamRedisSubscription(reader *bufio.Reader, writer *bufio.Wri func readRedisSubscriptionCommands(reader *bufio.Reader, commands chan<- redisSubscriptionCommand, done <-chan struct{}) { defer close(commands) + defer func() { + if recovered := recover(); recovered != nil { + log.Errorf("panic while reading redis subscription commands: %v", recovered) + } + }() for { args, errRead := readRESPArray(reader) @@ -423,12 +436,13 @@ func readRESPArray(reader *bufio.Reader) ([]string, error) { return nil, errLine } count, errParse := strconv.Atoi(line) - if errParse != nil || count < 0 { + if errParse != nil || count < 0 || count > maxRESPArrayLen { return nil, fmt.Errorf("protocol error") } args := make([]string, 0, count) + remainingBytes := maxRESPFrameBytes for i := 0; i < count; i++ { - value, errString := readRESPString(reader) + value, errString := readRESPStringWithBudget(reader, &remainingBytes) if errString != nil { return nil, errString } @@ -438,21 +452,39 @@ func readRESPArray(reader *bufio.Reader) ([]string, error) { } func readRESPString(reader *bufio.Reader) (string, error) { + return readRESPStringWithBudget(reader, nil) +} + +func readRESPStringWithBudget(reader *bufio.Reader, remainingBytes *int) (string, error) { prefix, errRead := reader.ReadByte() if errRead != nil { return "", errRead } switch prefix { case '$': - return readRESPBulkString(reader) + return readRESPBulkStringWithBudget(reader, remainingBytes) case '+', ':': - return readRESPLine(reader) + line, errLine := readRESPLine(reader) + if errLine != nil { + return "", errLine + } + if remainingBytes != nil { + if len(line) > *remainingBytes { + return "", fmt.Errorf("protocol error") + } + *remainingBytes -= len(line) + } + return line, nil default: return "", fmt.Errorf("protocol error") } } func readRESPBulkString(reader *bufio.Reader) (string, error) { + return readRESPBulkStringWithBudget(reader, nil) +} + +func readRESPBulkStringWithBudget(reader *bufio.Reader, remainingBytes *int) (string, error) { line, errLine := readRESPLine(reader) if errLine != nil { return "", errLine @@ -461,27 +493,56 @@ func readRESPBulkString(reader *bufio.Reader) (string, error) { if errParse != nil { return "", fmt.Errorf("protocol error") } + if length < -1 { + return "", fmt.Errorf("protocol error") + } if length < 0 { return "", nil } + if length > maxRESPBulkLen || (remainingBytes != nil && length > *remainingBytes) { + return "", fmt.Errorf("bulk string too large") + } buf := make([]byte, length+2) if _, errRead := io.ReadFull(reader, buf); errRead != nil { return "", errRead } - if length+2 < 2 || buf[length] != '\r' || buf[length+1] != '\n' { + if buf[length] != '\r' || buf[length+1] != '\n' { return "", fmt.Errorf("protocol error") } + if remainingBytes != nil { + *remainingBytes -= length + } return string(buf[:length]), nil } func readRESPLine(reader *bufio.Reader) (string, error) { - line, errRead := reader.ReadString('\n') - if errRead != nil { + var buf []byte + for { + fragment, errRead := reader.ReadSlice('\n') + if len(buf)+len(fragment) > maxRESPLineLen+2 { + return "", fmt.Errorf("protocol error") + } + if errRead == nil { + var line string + if len(buf) == 0 { + line = string(fragment) + } else { + buf = append(buf, fragment...) + line = string(buf) + } + line = strings.TrimSuffix(line, "\n") + line = strings.TrimSuffix(line, "\r") + if len(line) > maxRESPLineLen { + return "", fmt.Errorf("protocol error") + } + return line, nil + } + if errors.Is(errRead, bufio.ErrBufferFull) { + buf = append(buf, fragment...) + continue + } return "", errRead } - line = strings.TrimSuffix(line, "\n") - line = strings.TrimSuffix(line, "\r") - return line, nil } func writeRedisSimpleString(writer *bufio.Writer, value string) error { diff --git a/internal/api/response_compression.go b/internal/api/response_compression.go new file mode 100644 index 000000000..3c7181665 --- /dev/null +++ b/internal/api/response_compression.go @@ -0,0 +1,363 @@ +package api + +import ( + "bytes" + "compress/gzip" + "context" + "errors" + "io" + "net" + "net/http" + "strconv" + "strings" + "syscall" + + "github.com/gin-gonic/gin" + "github.com/klauspost/compress/zstd" + "github.com/router-for-me/CLIProxyAPI/v8/internal/config" + "github.com/router-for-me/CLIProxyAPI/v8/internal/httpwire" + log "github.com/sirupsen/logrus" +) + +func (s *Server) responseCompressionMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + responseCompressionMiddleware(s.getConfig())(c) + } +} + +func responseCompressionMiddleware(cfg *config.Config) gin.HandlerFunc { + return func(c *gin.Context) { + if cfg == nil || !cfg.EffectiveResponseCompression().IsEnabled() || + strings.EqualFold(strings.TrimSpace(c.GetHeader("Upgrade")), "websocket") { + c.Next() + return + } + original := c.Writer + commitWriter := &responseCompressionCommitWriter{ResponseWriter: original} + var writer *responseCompressionWriter + c.Writer = commitWriter + defer func() { + if recovered := recover(); recovered != nil { + // Discard uncommitted data so outer recovery can send a clean error. + if writer != nil && original.Written() { + writer.finish() + } + c.Writer = original + panic(recovered) + } + if writer != nil { + writer.finish() + } + // Commit even bodyless responses through the hook before restoring Gin's writer. + commitWriter.WriteHeaderNow() + c.Writer = original + }() + encoding, identityAllowed := negotiateResponseCompression(strings.Join(c.Request.Header.Values("Accept-Encoding"), ",")) + if encoding == "" || c.Request.Method == http.MethodHead { + if !identityAllowed { + log.Debug("response compression: no acceptable encoding; passing through for compatibility") + } + c.Next() + return + } + minBytes := cfg.EffectiveResponseCompression().EffectiveMinBytes() + if !identityAllowed { + minBytes = 0 + } + writer = &responseCompressionWriter{ + ResponseWriter: commitWriter, + encoding: encoding, + minBytes: minBytes, + } + c.Writer = writer + c.Next() + } +} + +// negotiateResponseCompression ranks quality first, using server preference for ties. +// The second result reports whether identity is allowed, including wildcard exclusions. +func negotiateResponseCompression(header string) (string, bool) { + qualities := make(map[string]float64) + for _, token := range strings.Split(header, ",") { + parts := strings.Split(token, ";") + name := strings.ToLower(strings.TrimSpace(parts[0])) + quality := 1.0 + for _, parameter := range parts[1:] { + key, value, ok := strings.Cut(parameter, "=") + if !ok || !strings.EqualFold(strings.TrimSpace(key), "q") { + continue + } + parsed, errParseFloat := strconv.ParseFloat(strings.TrimSpace(value), 64) + if errParseFloat != nil || !(parsed >= 0 && parsed <= 1) { + quality = 0 + } else { + quality = parsed + } + } + qualities[name] = quality + } + identityQuality, explicitIdentity := qualities["identity"] + wildcardQuality, explicitWildcard := qualities["*"] + identityAllowed := identityQuality > 0 || (!explicitIdentity && (!explicitWildcard || wildcardQuality > 0)) + selected := "" + highestQuality := 0.0 + for _, encoding := range []string{"zstd", "gzip"} { + quality, explicit := qualities[encoding] + if !explicit { + quality = wildcardQuality + } + if quality > highestQuality { + selected = encoding + highestQuality = quality + } + } + return selected, identityAllowed +} + +// responseCompressionCommitWriter merges Vary at the first transport write without buffering. +// Gin's WriteHeader only stages the status; Hijacker and other interfaces remain embedded. +type responseCompressionCommitWriter struct { + gin.ResponseWriter + committed bool +} + +func (w *responseCompressionCommitWriter) mergeVary() { + if !w.committed { + appendCompressionVary(w.Header()) + w.committed = true + } +} + +func (w *responseCompressionCommitWriter) Write(data []byte) (int, error) { + w.mergeVary() + return w.ResponseWriter.Write(data) +} + +func (w *responseCompressionCommitWriter) WriteString(data string) (int, error) { + w.mergeVary() + return w.ResponseWriter.WriteString(data) +} + +func (w *responseCompressionCommitWriter) WriteHeaderNow() { + w.mergeVary() + w.ResponseWriter.WriteHeaderNow() +} + +func (w *responseCompressionCommitWriter) Flush() { + if errFlush := w.FlushError(); errFlush != nil { + log.Debugf("response compression commit flush failed: %v", errFlush) + } +} + +func (w *responseCompressionCommitWriter) FlushError() error { + w.mergeVary() + return httpwire.FlushResponse(w.ResponseWriter) +} + +type responseCompressor interface { + io.WriteCloser + Flush() error +} + +// responseCompressionWriter keeps headers uncommitted until the encoding is known. +// Gin's remaining interfaces, including Hijacker and Pusher, are embedded unchanged. +type responseCompressionWriter struct { + gin.ResponseWriter + encoding string + minBytes int + buffer bytes.Buffer + compressor responseCompressor + decided bool + writeErr error + written bool + size int + status int +} + +func (w *responseCompressionWriter) Written() bool { + return w.written || w.ResponseWriter.Written() +} + +func (w *responseCompressionWriter) Size() int { + if !w.Written() { + return -1 + } + return w.size +} + +func (w *responseCompressionWriter) Status() int { + if w.status == 0 { + return w.ResponseWriter.Status() + } + return w.status +} + +func (w *responseCompressionWriter) WriteHeader(code int) { + if w.Written() { + return + } + w.ResponseWriter.WriteHeader(code) + w.status = w.ResponseWriter.Status() +} + +func (w *responseCompressionWriter) Write(data []byte) (n int, err error) { + if w.writeErr != nil { + return 0, w.writeErr + } + w.written = true + defer func() { w.size += n }() + if !w.decided { + if w.Header().Get("Content-Type") == "" && len(data) > 0 { + w.Header().Set("Content-Type", http.DetectContentType(data)) + } + w.decide(false, w.buffer.Len()+len(data)) + } + if !w.decided { + return w.buffer.Write(data) + } + if errDrain := w.drain(); errDrain != nil { + return 0, errDrain + } + if w.compressor != nil { + n, w.writeErr = w.compressor.Write(data) + } else { + n, w.writeErr = w.ResponseWriter.Write(data) + } + return n, w.writeErr +} + +// WriteString must not bypass the compression path through the embedded writer. +func (w *responseCompressionWriter) WriteString(data string) (int, error) { + return w.Write([]byte(data)) +} + +func (w *responseCompressionWriter) WriteHeaderNow() { + w.written = true + w.decide(true, w.buffer.Len()) + if errDrain := w.drain(); errDrain != nil { + log.Debugf("response compression header write failed: %v", errDrain) + } + w.ResponseWriter.WriteHeaderNow() +} + +func (w *responseCompressionWriter) Flush() { + if errFlush := w.FlushError(); errFlush != nil { + log.Debugf("response compression flush failed: %v", errFlush) + } +} + +// FlushError preserves transport errors for httpwire and streaming cancellation. +func (w *responseCompressionWriter) FlushError() error { + if w.writeErr != nil { + return w.writeErr + } + w.written = true + w.decide(true, w.buffer.Len()) + if errDrain := w.drain(); errDrain != nil { + return errDrain + } + if w.compressor != nil { + if errFlush := w.compressor.Flush(); errFlush != nil { + w.writeErr = errFlush + return errFlush + } + } + w.writeErr = httpwire.FlushResponse(w.ResponseWriter) + return w.writeErr +} + +func (w *responseCompressionWriter) decide(force bool, size int) { + if w.decided { + return + } + status := w.Status() + mediaType, _, _ := strings.Cut(w.Header().Get("Content-Type"), ";") + mediaType = strings.ToLower(strings.TrimSpace(mediaType)) + compressible := strings.HasPrefix(mediaType, "text/") || mediaType == "application/json" || + (strings.HasPrefix(mediaType, "application/") && strings.HasSuffix(mediaType, "+json")) || + mediaType == "application/javascript" || mediaType == "image/svg+xml" + if w.ResponseWriter.Written() || status < 200 || status == http.StatusNoContent || + status == http.StatusNotModified || status == http.StatusPartialContent || + w.Header().Get("Content-Encoding") != "" || !compressible { + w.decided = true + return + } + streaming := mediaType == "text/event-stream" + if !streaming { + if contentLength, errParseInt := strconv.ParseInt(w.Header().Get("Content-Length"), 10, 64); errParseInt == nil && contentLength >= 0 && contentLength < int64(w.minBytes) { + w.decided = true + return + } + if size < w.minBytes { + w.decided = force + return + } + } + var compressor responseCompressor + if w.encoding == "gzip" { + compressor = gzip.NewWriter(w.ResponseWriter) + } else { + encoder, errNewWriter := zstd.NewWriter(w.ResponseWriter, zstd.WithEncoderConcurrency(1)) + if errNewWriter != nil { + log.Errorf("create response compression encoder: %v", errNewWriter) + w.decided = true + return + } + compressor = encoder + } + w.compressor = compressor + w.decided = true + w.Header().Del("Content-Length") + w.Header().Set("Content-Encoding", w.encoding) + // Gin otherwise commits headers lazily when the encoder emits its first bytes. + w.ResponseWriter.WriteHeaderNow() +} + +func appendCompressionVary(header http.Header) { + seen := make(map[string]bool) + var tokens []string + for _, value := range header.Values("Vary") { + for _, token := range strings.Split(value, ",") { + token = strings.TrimSpace(token) + key := strings.ToLower(token) + if token != "" && !seen[key] { + tokens = append(tokens, token) + seen[key] = true + } + } + } + if !seen["accept-encoding"] { + tokens = append(tokens, "Accept-Encoding") + } + header.Set("Vary", strings.Join(tokens, ", ")) +} + +func (w *responseCompressionWriter) drain() error { + if w.writeErr != nil || w.buffer.Len() == 0 { + return w.writeErr + } + var target io.Writer = w.ResponseWriter + if w.compressor != nil { + target = w.compressor + } + _, w.writeErr = target.Write(w.buffer.Bytes()) + w.buffer.Reset() + return w.writeErr +} + +func (w *responseCompressionWriter) finish() { + w.decide(true, w.buffer.Len()) + if errDrain := w.drain(); errDrain != nil { + log.Debugf("response compression final write failed: %v", errDrain) + } + if w.compressor != nil { + if errClose := w.compressor.Close(); errClose != nil { + if w.writeErr != nil || errors.Is(errClose, context.Canceled) || errors.Is(errClose, net.ErrClosed) || + errors.Is(errClose, io.ErrClosedPipe) || errors.Is(errClose, syscall.EPIPE) || errors.Is(errClose, syscall.ECONNRESET) { + log.Debugf("response compression client disconnected: %v", errClose) + } else { + log.Errorf("close response compression encoder: %v", errClose) + } + } + } +} diff --git a/internal/api/server.go b/internal/api/server.go index 3cd910b83..87ddc279b 100644 --- a/internal/api/server.go +++ b/internal/api/server.go @@ -22,12 +22,14 @@ import ( "github.com/router-for-me/CLIProxyAPI/v8/internal/api/middleware" codexlive "github.com/router-for-me/CLIProxyAPI/v8/internal/client/codex/live" "github.com/router-for-me/CLIProxyAPI/v8/internal/config" + "github.com/router-for-me/CLIProxyAPI/v8/internal/githubauth" "github.com/router-for-me/CLIProxyAPI/v8/internal/logging" "github.com/router-for-me/CLIProxyAPI/v8/internal/managementasset" "github.com/router-for-me/CLIProxyAPI/v8/internal/pluginhost" "github.com/router-for-me/CLIProxyAPI/v8/internal/redisqueue" sdkaccess "github.com/router-for-me/CLIProxyAPI/v8/sdk/access" "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers" + "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers/openai" "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth" log "github.com/sirupsen/logrus" "golang.org/x/net/http2" @@ -118,6 +120,9 @@ type Server struct { // Returns: // - *Server: A new server instance func NewServer(cfg *config.Config, authManager *auth.Manager, accessManager *sdkaccess.Manager, configFilePath string, opts ...ServerOption) *Server { + if cfg != nil { + cfg = cfg.CloneForRuntime() + } optionState := &serverOptionConfig{ requestLoggerFactory: defaultRequestLoggerFactory, } @@ -141,10 +146,34 @@ func NewServer(cfg *config.Config, authManager *auth.Manager, accessManager *sdk optionState.engineConfigurator(engine) } + wd, err := os.Getwd() + if err != nil { + wd = configFilePath + } + + envAdminPassword, envAdminPasswordSet := os.LookupEnv("MANAGEMENT_PASSWORD") + envAdminPassword = strings.TrimSpace(envAdminPassword) + envManagementSecret := envAdminPasswordSet && envAdminPassword != "" + + // Create server instance + s := &Server{ + engine: engine, + handlers: handlers.NewBaseAPIHandlers(effectiveSDKConfig(cfg), authManager), + cfg: cfg, + accessManager: accessManager, + configFilePath: configFilePath, + currentPath: wd, + envManagementSecret: envManagementSecret, + wsRoutes: make(map[string]struct{}), + pluginHost: optionState.pluginHost, + exampleAPIKeySafeModeEnabled: optionState.exampleAPIKeySafeMode, + } + // Add middleware engine.Use(logging.GinLogrusLogger()) engine.Use(logging.GinLogrusRecovery()) engine.Use(logging.CPATraceIDMiddleware()) + engine.Use(s.maxRequestBodyBytesMiddleware()) for _, mw := range optionState.extraMiddleware { engine.Use(mw) } @@ -164,33 +193,12 @@ func NewServer(cfg *config.Config, authManager *auth.Manager, accessManager *sdk } } } + s.requestLogger = requestLogger + s.loggerToggle = toggle engine.Use(corsMiddleware()) - wd, err := os.Getwd() - if err != nil { - wd = configFilePath - } - - envAdminPassword, envAdminPasswordSet := os.LookupEnv("MANAGEMENT_PASSWORD") - envAdminPassword = strings.TrimSpace(envAdminPassword) - envManagementSecret := envAdminPasswordSet && envAdminPassword != "" - - // Create server instance - s := &Server{ - engine: engine, - handlers: handlers.NewBaseAPIHandlers(effectiveSDKConfig(cfg), authManager), - cfg: cfg, - accessManager: accessManager, - requestLogger: requestLogger, - loggerToggle: toggle, - configFilePath: configFilePath, - currentPath: wd, - envManagementSecret: envManagementSecret, - wsRoutes: make(map[string]struct{}), - pluginHost: optionState.pluginHost, - - exampleAPIKeySafeModeEnabled: optionState.exampleAPIKeySafeMode, - } + engine.Use(s.responseCompressionMiddleware()) + openai.SetResponsesWebsocketCompressionEnabled(cfg.Codex.WebsocketCompressionEnabled()) s.wsAuthEnabled.Store(cfg.WebsocketAuth) s.exampleAPIKeySafeModeActive.Store(s.exampleAPIKeySafeModeRequired(cfg)) s.handlers.SetPluginHost(optionState.pluginHost) @@ -204,10 +212,12 @@ func NewServer(cfg *config.Config, authManager *auth.Manager, accessManager *sdk if authManager != nil { authManager.SetRetryConfig(cfg.RequestRetry, time.Duration(cfg.MaxRetryInterval)*time.Second, cfg.MaxRetryCredentials) } + githubauth.SetToken(cfg.GitHubToken) managementasset.SetCurrentConfig(cfg) auth.SetQuotaCooldownDisabled(cfg.DisableCooling) auth.SetTransientErrorCooldownSeconds(cfg.TransientErrorCooldownSeconds) applySignatureCacheConfig(nil, cfg) + applyClaudeToolSearchConfig(cfg) // Initialize management handler s.mgmt = managementHandlers.NewHandler(cfg, configFilePath, authManager) s.mgmt.SetPluginHost(optionState.pluginHost) diff --git a/internal/api/server_management.go b/internal/api/server_management.go index 12a2ba537..1eabc9cdc 100644 --- a/internal/api/server_management.go +++ b/internal/api/server_management.go @@ -211,11 +211,12 @@ func (s *Server) managementAvailabilityMiddleware() gin.HandlerFunc { } func (s *Server) managementAvailable(c *gin.Context) bool { - if s == nil || s.cfg == nil { + cfg := s.getConfig() + if s == nil || cfg == nil { c.AbortWithStatus(http.StatusNotFound) return false } - if s.cfg.Home.Enabled { + if cfg.Home.Enabled { c.AbortWithStatus(http.StatusNotFound) return false } @@ -296,7 +297,8 @@ func (s *Server) pluginResourceNoRoute(c *gin.Context) { } return } - if s.cfg == nil || s.cfg.Home.Enabled || s.pluginHost == nil { + cfg := s.getConfig() + if cfg == nil || cfg.Home.Enabled || s.pluginHost == nil { c.AbortWithStatus(http.StatusNotFound) return } @@ -308,7 +310,7 @@ func (s *Server) pluginResourceNoRoute(c *gin.Context) { } func (s *Server) serveManagementControlPanel(c *gin.Context) { - cfg := s.cfg + cfg := s.getConfig() if cfg == nil || cfg.Home.Enabled || cfg.RemoteManagement.DisableControlPanel { c.AbortWithStatus(http.StatusNotFound) return diff --git a/internal/api/server_management_v8.go b/internal/api/server_management_v8.go index 7f68dad01..a0cb34369 100644 --- a/internal/api/server_management_v8.go +++ b/internal/api/server_management_v8.go @@ -11,8 +11,8 @@ import ( // Configuration paths mirror the v8 YAML tree; operational routes use its groups. func (s *Server) registerManagementV8Routes() { const prefix = "/v8/management" - s.engine.GET(prefix+"/oauth/callback", s.managementAvailabilityMiddleware(), s.mgmt.GetOAuthCallback) - s.engine.POST(prefix+"/oauth/callback", s.managementAvailabilityMiddleware(), s.mgmt.PostOAuthCallback) + s.engine.GET(prefix+"/oauth/callback", s.managementAvailabilityMiddleware(), s.mgmt.GetOAuthCallbackV8) + s.engine.POST(prefix+"/oauth/callback", s.managementAvailabilityMiddleware(), s.mgmt.PostOAuthCallbackV8) v8 := s.engine.Group(prefix) v8.Use(s.managementAvailabilityMiddleware(), s.mgmt.Middleware(), func(c *gin.Context) { @@ -41,8 +41,8 @@ func (s *Server) registerManagementV8Routes() { v8.GET("/observability/usage/queue", s.mgmt.GetUsageQueue) v8.GET("/credentials", s.mgmt.ListAuthFiles) - v8.POST("/credentials", s.mgmt.UploadAuthFile) - v8.DELETE("/credentials", s.mgmt.DeleteAuthFile) + v8.POST("/credentials", s.mgmt.UploadAuthFileV8) + v8.DELETE("/credentials", s.mgmt.DeleteAuthFileV8) v8.GET("/credentials/models", s.mgmt.GetAuthFileModels) v8.GET("/credentials/download", s.mgmt.DownloadAuthFile) v8.PATCH("/credentials/status", s.mgmt.PatchAuthFileStatus) @@ -50,6 +50,10 @@ func (s *Server) registerManagementV8Routes() { v8.POST("/credentials/refresh", s.mgmt.RefreshAuthFiles) v8.POST("/oauth/import", s.mgmt.ImportOAuthV8) v8.GET("/oauth/auth-url", s.mgmt.StartOAuthV8) + v8.POST("/oauth/auth-url", s.mgmt.StartOAuthV8) + v8.POST("/oauth/siwc/bind", s.mgmt.StartSIWCBind) + v8.GET("/oauth/siwc/status", s.mgmt.GetSIWCStatus) + v8.GET("/credentials/siwc/status", s.mgmt.GetSIWCStatus) v8.GET("/oauth/status", s.mgmt.GetAuthStatus) v8.DELETE("/oauth/session", s.mgmt.CancelAuthSession) diff --git a/internal/api/server_middleware.go b/internal/api/server_middleware.go index 3e75bedaf..6da5abc7b 100644 --- a/internal/api/server_middleware.go +++ b/internal/api/server_middleware.go @@ -11,6 +11,8 @@ import ( "github.com/router-for-me/CLIProxyAPI/v8/internal/logging" "github.com/router-for-me/CLIProxyAPI/v8/internal/safemode" sdkaccess "github.com/router-for-me/CLIProxyAPI/v8/sdk/access" + "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers" + coreusage "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/usage" log "github.com/sirupsen/logrus" ) @@ -26,20 +28,69 @@ var corsExposedResponseHeaders = []string{ "X-SERVER-BUILD-DATE", "Location", "Retry-After", + // Cross-origin management clients must read the config version token to + // send it back as If-Match on writes. + "ETag", "X-Request-Id", "OpenAI-Request-Id", } var corsExposedResponseHeadersJoined = strings.Join(corsExposedResponseHeaders, ", ") +// corsAllowedRequestHeaders lists the request headers browser clients may +// send to the API. The Fetch standard treats Authorization as a CORS +// non-wildcard request-header name, so it must be named explicitly: a +// wildcard does not cover it when a browser enforces that rule. +var corsAllowedRequestHeaders = []string{ + "Authorization", + "Content-Type", + "X-Api-Key", + "X-Goog-Api-Key", + "Anthropic-Version", + "Anthropic-Beta", +} + const ( exampleAPIKeyManagementPath = "/management.html" exampleAPIKeyManagementURL = "/management.html?safe-mode=configure" ) +func (s *Server) maxRequestBodyBytes() int64 { + if s == nil { + return 0 + } + cfg := s.getConfig() + if cfg == nil || cfg.MaxRequestBodyBytes <= 0 { + return 0 + } + return cfg.MaxRequestBodyBytes +} + +func (s *Server) maxRequestBodyBytesMiddleware() gin.HandlerFunc { + return func(c *gin.Context) { + maxBytes := s.maxRequestBodyBytes() + if maxBytes <= 0 || c == nil || c.Request == nil { + c.Next() + return + } + c.Set(handlers.MaxRequestBodyBytesContextKey, maxBytes) + if c.Request.ContentLength > maxBytes { + c.AbortWithStatusJSON(http.StatusRequestEntityTooLarge, gin.H{ + "error": "http: request body too large", + }) + return + } + if c.Request.Body != nil && c.Request.Body != http.NoBody { + c.Request.Body = http.MaxBytesReader(c.Writer, c.Request.Body, maxBytes) + } + c.Next() + } +} + func (s *Server) homeHeartbeatMiddleware() gin.HandlerFunc { return func(c *gin.Context) { - if s == nil || s.cfg == nil || !s.cfg.Home.Enabled { + cfg := s.getConfig() + if s == nil || cfg == nil || !cfg.Home.Enabled { c.Next() return } @@ -93,7 +144,7 @@ func (s *Server) exampleAPIKeySafeModeMiddleware() gin.HandlerFunc { } func (s *Server) serveExampleAPIKeyWarningPage(c *gin.Context) { - cfg := s.cfg + cfg := s.getConfig() var keys []string if cfg != nil { keys = safemode.ExampleAPIKeys(cfg.APIKeys) @@ -133,7 +184,7 @@ func corsMiddleware() gin.HandlerFunc { return func(c *gin.Context) { c.Header("Access-Control-Allow-Origin", "*") c.Header("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS") - c.Header("Access-Control-Allow-Headers", "*") + c.Header("Access-Control-Allow-Headers", corsAllowHeadersValue(c.Request)) c.Header("Access-Control-Expose-Headers", corsExposedResponseHeadersJoined) if c.Request.Method == "OPTIONS" { @@ -145,6 +196,39 @@ func corsMiddleware() gin.HandlerFunc { } } +// corsAllowHeadersValue builds the Access-Control-Allow-Headers value for a +// request. It always names the known API request headers explicitly and, for +// preflight requests, also echoes the headers the client asked about, so no +// allowed header depends on the wildcard that strict non-wildcard rules +// reject for Authorization. Echoing requested headers preserves the previous +// wildcard behavior for arbitrary custom client headers without widening +// the allowed origins or enabling credentials. +func corsAllowHeadersValue(req *http.Request) string { + allowed := make([]string, 0, len(corsAllowedRequestHeaders)) + seen := make(map[string]struct{}, len(corsAllowedRequestHeaders)) + appendHeader := func(name string) { + name = http.CanonicalHeaderKey(strings.TrimSpace(name)) + if name == "" { + return + } + key := strings.ToLower(name) + if _, ok := seen[key]; ok { + return + } + seen[key] = struct{}{} + allowed = append(allowed, name) + } + for _, name := range corsAllowedRequestHeaders { + appendHeader(name) + } + if req != nil && req.Method == http.MethodOptions { + for _, name := range strings.Split(req.Header.Get("Access-Control-Request-Headers"), ",") { + appendHeader(name) + } + } + return strings.Join(allowed, ", ") +} + // AuthMiddleware returns a Gin middleware handler that authenticates requests // using the configured authentication providers. When no providers are available, // it allows all requests (legacy behaviour). @@ -171,6 +255,10 @@ func accessAuthMiddleware(manager *sdkaccess.Manager, realtimeError bool) gin.Ha if len(result.Metadata) > 0 { c.Set("accessMetadata", result.Metadata) } + reqCtx := coreusage.WithAPIKey(c.Request.Context(), result.Principal) + reqCtx = coreusage.WithAccessProvider(reqCtx, result.Provider) + reqCtx = coreusage.WithIsNativeKey(reqCtx, coreusage.IsNativeAccessProvider(result.Provider)) + c.Request = c.Request.WithContext(reqCtx) } c.Next() return @@ -183,7 +271,10 @@ func accessAuthMiddleware(manager *sdkaccess.Manager, realtimeError bool) gin.Ha if realtimeError { errorType := "authentication_error" code := "invalid_api_key" - if statusCode >= http.StatusInternalServerError { + if statusCode == http.StatusRequestEntityTooLarge { + errorType = "invalid_request_error" + code = "request_body_too_large" + } else if statusCode >= http.StatusInternalServerError { errorType = "server_error" code = "authentication_service_error" } @@ -226,6 +317,10 @@ func realtimeAuthMiddleware(manager *sdkaccess.Manager, handler *codexlive.Handl } c.Set("userApiKey", principal) c.Set("accessProvider", provider) + reqCtx := coreusage.WithAPIKey(c.Request.Context(), principal) + reqCtx = coreusage.WithAccessProvider(reqCtx, provider) + reqCtx = coreusage.WithIsNativeKey(reqCtx, coreusage.IsNativeAccessProvider(provider)) + c.Request = c.Request.WithContext(reqCtx) c.Set(codexlive.ClientSecretSessionContextKey, authorization.Session) c.Set(codexlive.ClientSecretPrincipalContextKey, authorization.Principal) c.Next() diff --git a/internal/api/server_reload.go b/internal/api/server_reload.go index 64b7ee9bd..312182c0a 100644 --- a/internal/api/server_reload.go +++ b/internal/api/server_reload.go @@ -9,11 +9,14 @@ import ( "github.com/router-for-me/CLIProxyAPI/v8/internal/access" "github.com/router-for-me/CLIProxyAPI/v8/internal/cache" "github.com/router-for-me/CLIProxyAPI/v8/internal/config" + "github.com/router-for-me/CLIProxyAPI/v8/internal/githubauth" "github.com/router-for-me/CLIProxyAPI/v8/internal/logging" "github.com/router-for-me/CLIProxyAPI/v8/internal/managementasset" "github.com/router-for-me/CLIProxyAPI/v8/internal/redisqueue" "github.com/router-for-me/CLIProxyAPI/v8/internal/runtime/executor" + responsesclaude "github.com/router-for-me/CLIProxyAPI/v8/internal/translator/claude/openai/responses" "github.com/router-for-me/CLIProxyAPI/v8/internal/util" + "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers/openai" sdkAuth "github.com/router-for-me/CLIProxyAPI/v8/sdk/auth" "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth" log "github.com/sirupsen/logrus" @@ -52,10 +55,14 @@ func (s *Server) UpdateClientsContext(ctx context.Context, cfg *config.Config) b if errContext := ctx.Err(); errContext != nil { return false } + cfg = cfg.CloneForRuntime() // Reconstruct old config from YAML snapshot to avoid reference sharing issues var oldCfg *config.Config - if len(s.oldConfigYaml) > 0 { - _ = yaml.Unmarshal(s.oldConfigYaml, &oldCfg) + s.cfgMu.RLock() + oldConfigYaml := append([]byte(nil), s.oldConfigYaml...) + s.cfgMu.RUnlock() + if len(oldConfigYaml) > 0 { + _ = yaml.Unmarshal(oldConfigYaml, &oldCfg) } // Update request logger enabled state if it has changed @@ -112,6 +119,7 @@ func (s *Server) UpdateClientsContext(ctx context.Context, cfg *config.Config) b } applySignatureCacheConfig(oldCfg, cfg) + applyClaudeToolSearchConfig(cfg) if oldCfg != nil && !reflect.DeepEqual(oldCfg.Antigravity.ConnectionPool, cfg.Antigravity.ConnectionPool) { executor.ResetAntigravityTransports() @@ -176,15 +184,20 @@ func (s *Server) UpdateClientsContext(ctx context.Context, cfg *config.Config) b } } s.wsAuthEnabled.Store(cfg.WebsocketAuth) + openai.SetResponsesWebsocketCompressionEnabled(cfg.Codex.WebsocketCompressionEnabled()) if oldCfg != nil && s.wsAuthChanged != nil && oldCfg.WebsocketAuth != cfg.WebsocketAuth { s.wsAuthChanged(oldCfg.WebsocketAuth, cfg.WebsocketAuth) } + githubauth.SetToken(cfg.GitHubToken) managementasset.SetCurrentConfig(cfg) if errContext := ctx.Err(); errContext != nil { return false } // Save YAML snapshot for next comparison - s.oldConfigYaml, _ = yaml.Marshal(cfg) + marshaledCfg, _ := yaml.Marshal(cfg) + s.cfgMu.Lock() + s.oldConfigYaml = marshaledCfg + s.cfgMu.Unlock() s.handlers.UpdateClients(effectiveSDKConfig(cfg)) s.handlers.SetPluginHost(s.pluginHost) @@ -251,6 +264,13 @@ func (s *Server) SetWebsocketAuthChangeHandler(fn func(bool, bool)) { s.wsAuthChanged = fn } +// applyClaudeToolSearchConfig publishes upstream.claude.tool-search to the +// Claude Responses translator, which the Claude executor and the Codex model +// catalog also read so all three agree on it. +func applyClaudeToolSearchConfig(cfg *config.Config) { + responsesclaude.SetClientToolSearchEnabled(cfg != nil && cfg.Claude.ToolSearch) +} + func configuredSignatureCacheEnabled(cfg *config.Config) bool { if cfg != nil && cfg.AntigravitySignatureCacheEnabled != nil { return *cfg.AntigravitySignatureCacheEnabled diff --git a/internal/api/server_routes.go b/internal/api/server_routes.go index deb5c7c37..7bf7ff70b 100644 --- a/internal/api/server_routes.go +++ b/internal/api/server_routes.go @@ -24,6 +24,7 @@ import ( "github.com/router-for-me/CLIProxyAPI/v8/internal/logging" "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" "github.com/router-for-me/CLIProxyAPI/v8/internal/runtime/executor/helps" + responsesclaude "github.com/router-for-me/CLIProxyAPI/v8/internal/translator/claude/openai/responses" "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers" "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers/claude" "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers/gemini" @@ -57,6 +58,7 @@ func (s *Server) setupRoutes() { geminiHandlers := gemini.NewGeminiAPIHandler(s.handlers) claudeCodeHandlers := claude.NewClaudeCodeAPIHandler(s.handlers) openaiResponsesHandlers := openai.NewOpenAIResponsesAPIHandler(s.handlers) + openaiResponsesHandlers.AccessManager = s.accessManager s.codexLiveHandler = codexlive.NewHandler(s.handlers.AuthManager, s.cfg) // OpenAI compatible API routes @@ -64,6 +66,10 @@ func (s *Server) setupRoutes() { v1.Use(AuthMiddleware(s.accessManager)) { v1.GET("/models", s.unifiedModelsHandler(openaiHandlers, claudeCodeHandlers)) + v1.GET("/models/*model", func(c *gin.Context) { + c.Set(handlers.ModelDetailIDContextKey, strings.TrimPrefix(c.Param("model"), "/")) + s.unifiedModelsHandler(openaiHandlers, claudeCodeHandlers)(c) + }) v1.POST("/chat/completions", openaiHandlers.ChatCompletions) v1.POST("/completions", openaiHandlers.Completions) v1.POST("/images/generations", openaiHandlers.ImagesGenerations) @@ -73,6 +79,8 @@ func (s *Server) setupRoutes() { v1.POST("/videos/edits", openaiHandlers.XAIVideosEdits) v1.POST("/videos/extensions", openaiHandlers.XAIVideosExtensions) v1.GET("/videos/:request_id", openaiHandlers.XAIVideosRetrieve) + v1.POST("/audio/speech", openaiHandlers.AudioSpeech) + v1.POST("/tts", openaiHandlers.XAITTS) v1.POST("/messages", claudeCodeHandlers.ClaudeMessages) v1.POST("/messages/count_tokens", claudeCodeHandlers.ClaudeCountTokens) v1.GET("/responses", openaiResponsesHandlers.ResponsesWebsocket) @@ -151,7 +159,9 @@ func (s *Server) setupRoutes() { errStr = c.Query("error_description") } if state != "" { - _, _ = managementHandlers.WriteOAuthCallbackFileForPendingSession(s.cfg.AuthDir, "anthropic", state, code, errStr) + if cfg := s.getConfig(); cfg != nil { + _, _ = managementHandlers.WriteOAuthCallbackFileForPendingSession(cfg.AuthDir, "anthropic", state, code, errStr) + } } c.Header("Content-Type", "text/html; charset=utf-8") c.String(http.StatusOK, oauthCallbackSuccessHTML) @@ -165,7 +175,12 @@ func (s *Server) setupRoutes() { errStr = c.Query("error_description") } if state != "" { - _, _ = managementHandlers.WriteOAuthCallbackFileForPendingSession(s.cfg.AuthDir, "codex", state, code, errStr) + if cfg := s.getConfig(); cfg != nil { + if _, errWrite := managementHandlers.WriteOAuthCallbackFileForPendingSession(cfg.AuthDir, "codex", state, code, errStr); errors.Is(errWrite, managementHandlers.ErrSIWCCallbackRequiresV8) { + c.JSON(http.StatusConflict, gin.H{"error": "SIWC callbacks require /v8/management/oauth/callback"}) + return + } + } } c.Header("Content-Type", "text/html; charset=utf-8") c.String(http.StatusOK, oauthCallbackSuccessHTML) @@ -179,7 +194,9 @@ func (s *Server) setupRoutes() { errStr = c.Query("error_description") } if state != "" { - _, _ = managementHandlers.WriteOAuthCallbackFileForPendingSession(s.cfg.AuthDir, "antigravity", state, code, errStr) + if cfg := s.getConfig(); cfg != nil { + _, _ = managementHandlers.WriteOAuthCallbackFileForPendingSession(cfg.AuthDir, "antigravity", state, code, errStr) + } } c.Header("Content-Type", "text/html; charset=utf-8") c.String(http.StatusOK, oauthCallbackSuccessHTML) @@ -197,7 +214,12 @@ func (s *Server) setupRoutes() { c.JSON(http.StatusBadRequest, gin.H{"error": "code or error is required"}) return } - if _, errWrite := managementHandlers.WriteOAuthCallbackFileForPendingSession(s.cfg.AuthDir, "devin", state, code, errStr); errWrite != nil { + cfg := s.getConfig() + authDir := "" + if cfg != nil { + authDir = cfg.AuthDir + } + if _, errWrite := managementHandlers.WriteOAuthCallbackFileForPendingSession(authDir, "devin", state, code, errStr); errWrite != nil { c.JSON(http.StatusBadRequest, gin.H{"error": "invalid or expired OAuth callback"}) return } @@ -218,8 +240,10 @@ func (s *Server) codexAlphaSearchModelRouterHost() handlers.PluginModelRouterHos if s.pluginHost != nil { return s.pluginHost } - if s.handlers != nil && s.handlers.ModelRouterHost != nil { - return s.handlers.ModelRouterHost + if s.handlers != nil { + if host := s.handlers.ModelRouterHostSnapshot(); host != nil { + return host + } } return nil } @@ -458,7 +482,7 @@ func (s *Server) codexAlphaSearch(c *gin.Context) { return nil, errRequest } authType, authValue := current.AccountInfo() - helps.RecordAPIRequest(ctx, s.cfg, helps.UpstreamRequestLog{ + helps.RecordAPIRequest(ctx, s.getConfig(), helps.UpstreamRequestLog{ URL: upstreamURL, Method: http.MethodPost, Headers: req.Header.Clone(), @@ -479,6 +503,7 @@ func (s *Server) codexAlphaSearch(c *gin.Context) { c.JSON(clienterror.HTTPStatusFromErrorOr(errCtx, http.StatusRequestTimeout), gin.H{"error": errCtx.Error()}) return } + cfg := s.getConfig() resp, err := performRequest(selected) if err != nil { if errors.Is(err, errMissingBaseURL) { @@ -491,7 +516,7 @@ func (s *Server) codexAlphaSearch(c *gin.Context) { if selection != nil { selection.End("request_failed") } - helps.RecordAPIResponseError(ctx, s.cfg, err) + helps.RecordAPIResponseError(ctx, cfg, err) c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadGateway), gin.H{"error": err.Error()}) return } @@ -515,18 +540,18 @@ func (s *Server) codexAlphaSearch(c *gin.Context) { } else { defer func() { _ = closeResponseBody() }() } - helps.RecordAPIResponseMetadata(ctx, s.cfg, resp.StatusCode, resp.Header.Clone()) + helps.RecordAPIResponseMetadata(ctx, cfg, resp.StatusCode, resp.Header.Clone()) upstreamBody, err := io.ReadAll(io.LimitReader(resp.Body, 32<<20)) if err != nil { - helps.AppendAPIResponseChunk(ctx, s.cfg, upstreamBody) + helps.AppendAPIResponseChunk(ctx, cfg, upstreamBody) if selection != nil && resp.StatusCode == http.StatusUnauthorized { s.handlers.AuthManager.ReportHomeUnauthorized(ctx, selected, "codex", selectionModel, upstreamBody) } - helps.RecordAPIResponseError(ctx, s.cfg, err) + helps.RecordAPIResponseError(ctx, cfg, err) c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadGateway), gin.H{"error": "Failed to read Codex search response"}) return } - helps.AppendAPIResponseChunk(ctx, s.cfg, upstreamBody) + helps.AppendAPIResponseChunk(ctx, cfg, upstreamBody) if selection != nil && resp.StatusCode == http.StatusUnauthorized { s.handlers.AuthManager.ReportHomeUnauthorized(ctx, selected, "codex", selectionModel, upstreamBody) log.WithField("status", resp.StatusCode).Warnf("codex alpha search upstream request failed: %s", logging.SafeDiagnosticForLog(string(upstreamBody))) @@ -596,9 +621,10 @@ func (s *Server) unifiedModelsHandler(openaiHandler *openai.OpenAIAPIHandler, cl return } + cfg := s.getConfig() if _, ok := c.Request.URL.Query()["client_version"]; ok { clientVersion := c.Query("client_version") - if s != nil && s.cfg != nil && s.cfg.Home.Enabled { + if cfg != nil && cfg.Home.Enabled { s.handleHomeCodexClientModels(c, clientVersion) return } @@ -606,7 +632,7 @@ func (s *Server) unifiedModelsHandler(openaiHandler *openai.OpenAIAPIHandler, cl return } - if s != nil && s.cfg != nil && s.cfg.Home.Enabled { + if cfg != nil && cfg.Home.Enabled { s.handleHomeModels(c) return } @@ -653,7 +679,7 @@ func grokModelsFromRegistryInfos(infos []*registry.ModelInfo) []grokbuild.ModelI func (s *Server) handleGrokModels(c *gin.Context) { var models []grokbuild.ModelInfo - if s != nil && s.cfg != nil && s.cfg.Home.Enabled { + if cfg := s.getConfig(); cfg != nil && cfg.Home.Enabled { entries, ok := s.loadHomeModelEntries(c) if !ok { return @@ -681,9 +707,10 @@ func (s *Server) handleHomeCodexClientModels(c *gin.Context, clientVersion strin return } + cfg := s.getConfig() models := make([]map[string]any, 0, len(entries)) for _, entry := range entries { - models = append(models, formatHomeCodexModelWithSettings(entry, s.cfg)) + models = append(models, formatHomeCodexModelWithSettings(entry, cfg)) } var webSearchCapabilityForModel codexmodels.WebSearchCapabilityForModelFunc @@ -695,10 +722,18 @@ func (s *Server) handleHomeCodexClientModels(c *gin.Context, clientVersion strin manager = s.handlers.AuthManager } var applyPatchCapabilityForModel codexmodels.ApplyPatchCapabilityForModelFunc - if s.cfg.Client.Codex.EnableApplyPatch { + if cfg != nil && cfg.Client.Codex.EnableApplyPatch { applyPatchCapabilityForModel = homeApplyPatchCapabilityForModel(entries, manager) } - payload := codexmodels.BuildResponseForClientWithToolCapabilities(models, nil, webSearchCapabilityForModel, applyPatchCapabilityForModel, s.cfg.Client.Codex.OptimizeMultiAgentV2, clientVersion) + optimizeMultiAgentV2 := cfg != nil && cfg.Client.Codex.OptimizeMultiAgentV2 + var payload map[string]any + if responsesclaude.ClientToolSearchEnabled() { + payload = codexmodels.BuildResponseForClientWithSearchToolRoutes(models, nil, homeProvidersForModel(entries), webSearchCapabilityForModel, applyPatchCapabilityForModel, homeToolSearchCapabilityForModel(entries), optimizeMultiAgentV2, clientVersion) + } else { + // upstream.claude.tool-search off keeps the catalog that predates + // tool_search, which never consulted Home routes for search support. + payload = codexmodels.BuildResponseForClientWithToolCapabilities(models, nil, webSearchCapabilityForModel, applyPatchCapabilityForModel, nil, optimizeMultiAgentV2, clientVersion) + } body, errMarshal := codexmodels.MarshalCompact(payload) if errMarshal != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": errMarshal.Error()}) @@ -710,6 +745,15 @@ func (s *Server) handleHomeCodexClientModels(c *gin.Context, clientVersion strin // homeApplyPatchCapabilityForModel uses only Home's exact public routing evidence, // never the local model registry or the template's metadata model ID. func homeApplyPatchCapabilityForModel(entries []homeModelEntry, manager *auth.Manager) codexmodels.ApplyPatchCapabilityForModelFunc { + providersForModel := homeProvidersForModel(entries) + return func(id string) bool { + return manager.SupportsApplyPatchForProviders(providersForModel(id)) + } +} + +// homeProvidersForModel returns Home's routing providers for an exact public +// model ID. An empty entry stands for an unknown route so it can veto support. +func homeProvidersForModel(entries []homeModelEntry) codexmodels.ProvidersForModelFunc { providersByID := make(map[string][]string, len(entries)) for _, entry := range entries { id := strings.TrimSpace(entry.id) @@ -725,8 +769,31 @@ func homeApplyPatchCapabilityForModel(entries []homeModelEntry, manager *auth.Ma } } } + return func(id string) []string { + return providersByID[strings.TrimSpace(id)] + } +} + +// homeToolSearchCapabilityForModel trusts Home's per-route tool_search flag and +// requires it on every route of the model. A Home that predates the flag never +// enables native tool search. +func homeToolSearchCapabilityForModel(entries []homeModelEntry) codexmodels.ToolSearchCapabilityForModelFunc { + routesByID := make(map[string][]registry.NativeCapabilityRoute, len(entries)) + for _, entry := range entries { + id := strings.TrimSpace(entry.id) + routesByID[id] = append(routesByID[id], entry.nativeCapabilityRoutes...) + } return func(id string) bool { - return manager.SupportsApplyPatchForProviders(providersByID[strings.TrimSpace(id)]) + routes := routesByID[strings.TrimSpace(id)] + if len(routes) == 0 { + return false + } + for _, route := range routes { + if route.NativeCapabilities == nil || route.NativeCapabilities.ToolSearch == nil || !*route.NativeCapabilities.ToolSearch { + return false + } + } + return true } } @@ -805,7 +872,7 @@ func formatHomeCodexModelWithSettings(entry homeModelEntry, cfg *config.Config) func (s *Server) geminiModelsHandler(geminiHandler *gemini.GeminiAPIHandler) gin.HandlerFunc { return func(c *gin.Context) { - if s != nil && s.cfg != nil && s.cfg.Home.Enabled { + if cfg := s.getConfig(); cfg != nil && cfg.Home.Enabled { s.handleHomeGeminiModels(c) return } @@ -816,7 +883,7 @@ func (s *Server) geminiModelsHandler(geminiHandler *gemini.GeminiAPIHandler) gin func (s *Server) geminiGetHandler(geminiHandler *gemini.GeminiAPIHandler) gin.HandlerFunc { return func(c *gin.Context) { - if s != nil && s.cfg != nil && s.cfg.Home.Enabled { + if cfg := s.getConfig(); cfg != nil && cfg.Home.Enabled { s.handleHomeGeminiModel(c) return } @@ -847,7 +914,8 @@ func (s *Server) handleHomeModels(c *gin.Context) { isClaude := isAnthropicModelsRequest(c) if isClaude { - disableCloaking := s.cfg != nil && s.cfg.ClaudeCode.DisableCloakingModelList + cfg := s.getConfig() + disableCloaking := cfg != nil && cfg.ClaudeCode.DisableCloakingModelList s.writeModelListResponse(c, "claude", claudemodels.BuildResponse(formatHomeClaudeModels(entries), disableCloaking)) return } @@ -1009,12 +1077,19 @@ func formatHomeGeminiModel(entry homeModelEntry) map[string]any { if displayName == "" { displayName = entry.id } - return map[string]any{ + model := map[string]any{ "name": name, "displayName": displayName, "description": displayName, "supportedGenerationMethods": []string{"generateContent"}, } + if entry.contextLength > 0 { + model["inputTokenLimit"] = entry.contextLength + } + if entry.maxCompletionTokens > 0 { + model["outputTokenLimit"] = entry.maxCompletionTokens + } + return model } func homeGeminiModelMatches(entry homeModelEntry, action string) bool { @@ -1100,13 +1175,29 @@ func decodeHomeModels(raw []byte) ([]homeModelEntry, error) { if err := json.Unmarshal(raw, &bySection); err != nil { return nil, fmt.Errorf("parse home models payload: %w", err) } - if len(bySection) == 0 { - return nil, fmt.Errorf("home models payload has no sections") + if bySection == nil { + return nil, fmt.Errorf("home models payload is not an object") } + // The error envelope must never be interpreted as a provider model section. + if _, hasError := bySection["error"]; hasError { + return nil, fmt.Errorf("home models payload contains an error envelope") + } + + sections := make([]string, 0, len(bySection)) + for section := range bySection { + sections = append(sections, section) + } + sort.Strings(sections) indexByID := make(map[string]int) out := make([]homeModelEntry, 0, 256) - for section, models := range bySection { + hasModelRecords := false + for _, section := range sections { + models := bySection[section] + if models == nil { + return nil, fmt.Errorf("home models section %q is null", section) + } + hasModelRecords = hasModelRecords || len(models) > 0 provider := strings.ToLower(strings.TrimSpace(section)) for _, model := range models { id, _ := model["id"].(string) @@ -1124,12 +1215,7 @@ func decodeHomeModels(raw []byte) ([]homeModelEntry, error) { Provider: provider, NativeCapabilities: nativeCapabilities, } - if index, ok := indexByID[id]; ok { - out[index].providers = appendUniqueHomeProvider(out[index].providers, provider) - out[index].nativeCapabilityRoutes = append(out[index].nativeCapabilityRoutes, route) - continue - } - + created := homeModelInt64Value(model, "created") ownedBy, _ := model["owned_by"].(string) ownedBy = strings.TrimSpace(ownedBy) displayName, _ := model["display_name"].(string) @@ -1138,17 +1224,47 @@ func decodeHomeModels(raw []byte) ([]homeModelEntry, error) { displayName, _ = model["displayName"].(string) displayName = strings.TrimSpace(displayName) } + contextLength := int(homeModelInt64Value(model, "context_length", "contextLength", "inputTokenLimit", "max_input_tokens")) + maxContextLength := int(homeModelInt64Value(model, "max_context_length", "maxContextLength")) + maxCompletionTokens := int(homeModelInt64Value(model, "max_completion_tokens", "maxCompletionTokens", "outputTokenLimit", "max_tokens")) thinking := homeModelThinkingSupport(model) + if index, ok := indexByID[id]; ok { + out[index].providers = appendUniqueHomeProvider(out[index].providers, provider) + out[index].nativeCapabilityRoutes = append(out[index].nativeCapabilityRoutes, route) + if out[index].created <= 0 && created > 0 { + out[index].created = created + } + if out[index].ownedBy == "" && ownedBy != "" { + out[index].ownedBy = ownedBy + } + if out[index].displayName == "" && displayName != "" { + out[index].displayName = displayName + } + if out[index].contextLength <= 0 && contextLength > 0 { + out[index].contextLength = contextLength + } + if out[index].maxContextLength <= 0 && maxContextLength > 0 { + out[index].maxContextLength = maxContextLength + } + if out[index].maxCompletionTokens <= 0 && maxCompletionTokens > 0 { + out[index].maxCompletionTokens = maxCompletionTokens + } + if out[index].thinking == nil && thinking != nil { + out[index].thinking = thinking + } + continue + } + indexByID[id] = len(out) out = append(out, homeModelEntry{ id: id, - created: homeModelInt64Value(model, "created"), + created: created, ownedBy: ownedBy, displayName: displayName, - contextLength: int(homeModelInt64Value(model, "context_length", "contextLength", "inputTokenLimit", "max_input_tokens")), - maxContextLength: int(homeModelInt64Value(model, "max_context_length", "maxContextLength")), - maxCompletionTokens: int(homeModelInt64Value(model, "max_completion_tokens", "maxCompletionTokens", "outputTokenLimit", "max_tokens")), + contextLength: contextLength, + maxContextLength: maxContextLength, + maxCompletionTokens: maxCompletionTokens, thinking: thinking, providers: appendUniqueHomeProvider(nil, provider), nativeCapabilityRoutes: []registry.NativeCapabilityRoute{route}, @@ -1157,7 +1273,9 @@ func decodeHomeModels(raw []byte) ([]homeModelEntry, error) { } sort.Slice(out, func(i, j int) bool { return out[i].id < out[j].id }) - if len(out) == 0 { + // An empty catalog is valid when the client's access scopes have no routes. + // Non-empty sections with no usable model IDs still indicate malformed data. + if len(out) == 0 && hasModelRecords { return nil, fmt.Errorf("home models payload contains no models") } return out, nil @@ -1168,11 +1286,14 @@ func homeModelNativeCapabilities(model map[string]any) *registry.NativeCapabilit if !ok { return nil } - webSearch, ok := raw["web_search"].(bool) - if !ok { - return ®istry.NativeCapabilities{} + capabilities := ®istry.NativeCapabilities{} + if webSearch, okWebSearch := raw["web_search"].(bool); okWebSearch { + capabilities.WebSearch = &webSearch + } + if toolSearch, okToolSearch := raw["tool_search"].(bool); okToolSearch { + capabilities.ToolSearch = &toolSearch } - return ®istry.NativeCapabilities{WebSearch: &webSearch} + return capabilities } func appendUniqueHomeProvider(providers []string, provider string) []string { diff --git a/sdk/api/handlers/claude/code_handlers.go b/sdk/api/handlers/claude/code_handlers.go index 9a576e886..823decd66 100644 --- a/sdk/api/handlers/claude/code_handlers.go +++ b/sdk/api/handlers/claude/code_handlers.go @@ -11,6 +11,7 @@ import ( "compress/gzip" "context" "encoding/json" + "errors" "fmt" "io" "net/http" @@ -19,6 +20,7 @@ import ( "github.com/gin-gonic/gin" claudemodels "github.com/router-for-me/CLIProxyAPI/v8/internal/client/claude/models" + "github.com/router-for-me/CLIProxyAPI/v8/internal/clienterror" . "github.com/router-for-me/CLIProxyAPI/v8/internal/constant" "github.com/router-for-me/CLIProxyAPI/v8/internal/interfaces" "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" @@ -72,7 +74,7 @@ func (h *ClaudeCodeAPIHandler) ClaudeMessages(c *gin.Context) { rawJSON, err := c.GetRawData() // If data retrieval fails, return a 400 Bad Request error. if err != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ + c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), handlers.ErrorResponse{ Error: handlers.ErrorDetail{ Message: fmt.Sprintf("Invalid request: %v", err), Type: "invalid_request_error", @@ -104,7 +106,7 @@ func (h *ClaudeCodeAPIHandler) ClaudeCountTokens(c *gin.Context) { rawJSON, err := c.GetRawData() // If data retrieval fails, return a 400 Bad Request error. if err != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ + c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), handlers.ErrorResponse{ Error: handlers.ErrorDetail{ Message: fmt.Sprintf("Invalid request: %v", err), Type: "invalid_request_error", @@ -155,7 +157,8 @@ func rewriteClaudeDDModelInBody(rawJSON []byte) []byte { // Parameters: // - c: The Gin context for the request. func (h *ClaudeCodeAPIHandler) ClaudeModels(c *gin.Context) { - disableCloaking := h.Cfg != nil && h.Cfg.ClaudeCode.DisableCloakingModelList + cfg := h.CurrentConfig() + disableCloaking := cfg != nil && cfg.ClaudeCode.DisableCloakingModelList h.WriteModelListResponse(c, h.HandlerType(), claudemodels.BuildResponse(h.Models(), disableCloaking)) } @@ -237,13 +240,38 @@ func (h *ClaudeCodeAPIHandler) handleStreamingResponse(c *gin.Context, rawJSON [ // This allows proper cleanup and cancellation of ongoing requests cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) - dataChan, upstreamHeaders, errChan := h.ExecuteStreamWithAuthManager(cliCtx, h.HandlerType(), modelName, rawJSON, "") setSSEHeaders := func() { c.Header("Content-Type", "text/event-stream") c.Header("Cache-Control", "no-cache") c.Header("Connection", "keep-alive") c.Header("Access-Control-Allow-Origin", "*") } + var upstreamHeaders http.Header + committed := false + commitSSE := func() { + if committed { + return + } + setSSEHeaders() + handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + committed = true + } + writeKeepAlive := func() bool { + commitSSE() + _, _ = c.Writer.Write([]byte(": keep-alive\n\n")) + flusher.Flush() + return true + } + keepAliveC, stopKeepAlive := handlers.NewPreFirstTokenKeepAliveTicker(h.CurrentConfig()) + defer stopKeepAlive() + + dataChan, resolvedHeaders, errChan, okBootstrap := h.ExecuteStreamWithPreFTKeepAlive( + c, cliCtx, func(err error) { cliCancel(err) }, h.HandlerType(), modelName, rawJSON, "", keepAliveC, writeKeepAlive, + ) + if !okBootstrap { + return + } + upstreamHeaders = resolvedHeaders // Peek at the first chunk to determine success or failure before setting headers for { @@ -257,6 +285,14 @@ func (h *ClaudeCodeAPIHandler) handleStreamingResponse(c *gin.Context, rawJSON [ errChan = nil continue } + if committed { + stopKeepAlive() + pendingErrors := make(chan *interfaces.ErrorMessage, 1) + pendingErrors <- errMsg + close(pendingErrors) + h.forwardClaudeStream(c, flusher, func(err error) { cliCancel(err) }, make(chan []byte), pendingErrors) + return + } // Upstream failed immediately. Return proper error status and JSON. h.WriteErrorResponse(c, errMsg) if errMsg != nil { @@ -267,7 +303,15 @@ func (h *ClaudeCodeAPIHandler) handleStreamingResponse(c *gin.Context, rawJSON [ return case chunk, ok := <-dataChan: if !ok { + stopKeepAlive() if errMsg, hasPendingError := handlers.PendingStreamError(errChan); hasPendingError { + if committed { + pendingErrors := make(chan *interfaces.ErrorMessage, 1) + pendingErrors <- errMsg + close(pendingErrors) + h.forwardClaudeStream(c, flusher, func(err error) { cliCancel(err) }, make(chan []byte), pendingErrors) + return + } h.WriteErrorResponse(c, errMsg) if errMsg != nil { cliCancel(errMsg.Error) @@ -276,58 +320,211 @@ func (h *ClaudeCodeAPIHandler) handleStreamingResponse(c *gin.Context, rawJSON [ } return } - // Stream closed without data? Send DONE or just headers. - setSSEHeaders() - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) - flusher.Flush() - cliCancel(nil) + // Stream closed without data: delegate to forwardClaudeStream with the closed dataChan to surface missing message_stop. + commitSSE() + h.forwardClaudeStream(c, flusher, func(err error) { cliCancel(err) }, dataChan, errChan) return } // Success! Set headers now. - setSSEHeaders() - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + stopKeepAlive() + commitSSE() // Write the first chunk + var streamState claudeSSEStreamState + streamState.trailingNewlines = 2 + streamState.ObserveChunk(chunk) if len(chunk) > 0 { _, _ = c.Writer.Write(chunk) flusher.Flush() } // Continue streaming the rest - h.forwardClaudeStream(c, flusher, func(err error) { cliCancel(err) }, dataChan, errChan) + h.forwardClaudeStreamWithState(c, flusher, func(err error) { cliCancel(err) }, dataChan, errChan, &streamState) return + case <-keepAliveC: + writeKeepAlive() + } + } +} + +type claudeSSEStreamState struct { + sawErrorEvent bool + sawMessageStop bool + sawBareMessageJSON bool + trailingNewlines int + currentEvent string + eventData [][]byte + lineBuf bytes.Buffer +} + +func (s *claudeSSEStreamState) ObserveChunk(chunk []byte) { + if len(chunk) == 0 { + return + } + s.trailingNewlines = advanceClaudeSSENewlineCount(s.trailingNewlines, chunk) + for _, b := range chunk { + if b == '\n' { + s.observeLine(s.lineBuf.Bytes()) + s.lineBuf.Reset() + continue + } + s.lineBuf.WriteByte(b) + } +} + +func (s *claudeSSEStreamState) FlushPendingLine() { + if s.lineBuf.Len() > 0 { + s.observeLine(s.lineBuf.Bytes()) + s.lineBuf.Reset() + } + s.flushEventData() +} + +func (s *claudeSSEStreamState) flushEventData() { + if len(s.eventData) > 0 { + joined := bytes.TrimSpace(bytes.Join(s.eventData, []byte("\n"))) + s.eventData = s.eventData[:0] + if len(joined) > 0 && gjson.ValidBytes(joined) { + s.observeJSONPayload(joined, true) + } + } +} + +func (s *claudeSSEStreamState) ensureSSEEventBoundary(w io.Writer) { + if s.sawBareMessageJSON { + return + } + switch s.trailingNewlines { + case 0: + _, _ = w.Write([]byte("\n\n")) + case 1: + _, _ = w.Write([]byte("\n")) + } + s.trailingNewlines = 2 +} + +func (s *claudeSSEStreamState) observeLine(line []byte) { + trimmed := bytes.TrimSpace(line) + if len(trimmed) == 0 { + s.flushEventData() + s.currentEvent = "" + return + } + if bytes.HasPrefix(trimmed, []byte("event:")) { + s.currentEvent = strings.TrimSpace(string(trimmed[len("event:"):])) + return + } + if bytes.HasPrefix(trimmed, []byte("data:")) { + payload := bytes.TrimSpace(trimmed[len("data:"):]) + if len(payload) > 0 { + s.eventData = append(s.eventData, append([]byte(nil), payload...)) } + return + } + if !bytes.HasPrefix(trimmed, []byte("{")) || !gjson.ValidBytes(trimmed) { + return } + s.observeJSONPayload(trimmed, false) } -func (h *ClaudeCodeAPIHandler) forwardClaudeStream(c *gin.Context, flusher http.Flusher, cancel func(error), data <-chan []byte, errs <-chan *interfaces.ErrorMessage) { +func (s *claudeSSEStreamState) observeJSONPayload(payload []byte, hasDataPrefix bool) { + root := gjson.ParseBytes(payload) + switch root.Get("type").String() { + case "message_stop": + s.sawMessageStop = true + case "error": + s.sawErrorEvent = true + case "message": + if !hasDataPrefix && strings.TrimSpace(root.Get("stop_reason").String()) != "" { + s.sawMessageStop = true + s.sawBareMessageJSON = true + } + default: + if hasDataPrefix && s.currentEvent == "error" && root.Get("error").Exists() { + s.sawErrorEvent = true + } + } +} + +func advanceClaudeSSENewlineCount(prev int, chunk []byte) int { + count := prev + for _, b := range chunk { + switch b { + case '\r': + continue + case '\n': + if count < 2 { + count++ + } + default: + count = 0 + } + } + return count +} + +func (h *ClaudeCodeAPIHandler) forwardClaudeStream(c *gin.Context, flusher http.Flusher, cancel func(error), data <-chan []byte, errs <-chan *interfaces.ErrorMessage, initialErrorWritten ...bool) { + state := &claudeSSEStreamState{ + sawErrorEvent: len(initialErrorWritten) > 0 && initialErrorWritten[0], + trailingNewlines: 2, + } + h.forwardClaudeStreamWithState(c, flusher, cancel, data, errs, state) +} + +func (h *ClaudeCodeAPIHandler) forwardClaudeStreamWithState(c *gin.Context, flusher http.Flusher, cancel func(error), data <-chan []byte, errs <-chan *interfaces.ErrorMessage, state *claudeSSEStreamState) { + if state == nil { + state = &claudeSSEStreamState{trailingNewlines: 2} + } h.ForwardStream(c, flusher, cancel, data, errs, handlers.StreamForwardOptions{ WriteChunk: func(chunk []byte) { if len(chunk) == 0 { return } + state.ObserveChunk(chunk) _, _ = c.Writer.Write(chunk) }, WriteTerminalError: func(errMsg *interfaces.ErrorMessage) { + state.FlushPendingLine() if errMsg == nil { return } - status := http.StatusInternalServerError - if errMsg.StatusCode > 0 { - status = errMsg.StatusCode + if state.sawErrorEvent { + state.ensureSSEEventBoundary(c.Writer) + return } - c.Status(status) + state.ensureSSEEventBoundary(c.Writer) errorBytes, _ := json.Marshal(h.toClaudeError(errMsg)) _, _ = fmt.Fprintf(c.Writer, "event: error\ndata: %s\n\n", errorBytes) }, + CloseError: func() *interfaces.ErrorMessage { + state.FlushPendingLine() + if state.sawMessageStop || state.sawErrorEvent { + return nil + } + return &interfaces.ErrorMessage{ + StatusCode: http.StatusBadGateway, + Error: fmt.Errorf("upstream stream closed before message_stop"), + } + }, + WriteDone: func() { + state.FlushPendingLine() + if state.sawMessageStop || state.sawErrorEvent { + state.ensureSSEEventBoundary(c.Writer) + } + }, }) } type claudeErrorDetail struct { - Type string `json:"type"` - Message string `json:"message"` + Type string `json:"type"` + Message string `json:"message"` + Details *claudeErrorDetails `json:"details,omitempty"` +} + +type claudeErrorDetails struct { + ErrorCode string `json:"error_code"` } type claudeErrorResponse struct { @@ -344,19 +541,23 @@ func (h *ClaudeCodeAPIHandler) toClaudeError(msg *interfaces.ErrorMessage) claud errText = http.StatusText(status) } if msg.Error != nil { - if v := strings.TrimSpace(msg.Error.Error()); v != "" { + if v := strings.TrimSpace(claudeErrorText(msg.Error)); v != "" { errText = v } } } errType, message := claudeErrorDetailFromText(status, errText) - return claudeErrorResponse{ + response := claudeErrorResponse{ Type: "error", Error: claudeErrorDetail{ Type: errType, Message: message, }, } + if clienterror.IsClaudeThreadNotFound(status, errors.New(errText)) { + response.Error.Details = &claudeErrorDetails{ErrorCode: "thread_not_found"} + } + return response } func (h *ClaudeCodeAPIHandler) WriteErrorResponse(c *gin.Context, msg *interfaces.ErrorMessage) { @@ -375,6 +576,14 @@ func (h *ClaudeCodeAPIHandler) WriteErrorResponse(c *gin.Context, msg *interface } } body := bytes.Clone(msg.Body) + if clienterror.IsClaudeThreadNotFound(status, errors.New(string(body))) { + if markedBody, errMarshal := json.Marshal(h.toClaudeError(&interfaces.ErrorMessage{ + StatusCode: status, + Error: errors.New(string(body)), + })); errMarshal == nil { + body = markedBody + } + } appendClaudeAPIResponse(c, body) if !c.Writer.Written() && c.Writer.Header().Get("Content-Type") == "" { c.Writer.Header().Set("Content-Type", "application/json") @@ -388,7 +597,7 @@ func (h *ClaudeCodeAPIHandler) WriteErrorResponse(c *gin.Context, msg *interface c.Writer.Header().Add("Retry-After", value) } } - if msg != nil && msg.Addon != nil && handlers.PassthroughHeadersEnabled(h.Cfg) { + if msg != nil && msg.Addon != nil && handlers.PassthroughHeadersEnabled(h.CurrentConfig()) { for key, values := range msg.Addon { if len(values) == 0 || handlers.IsCPAReservedResponseHeader(key) { continue @@ -412,6 +621,20 @@ func (h *ClaudeCodeAPIHandler) WriteErrorResponse(c *gin.Context, msg *interface _, _ = c.Writer.Write(body) } +func claudeErrorText(err error) string { + if err == nil { + return "" + } + type responseBodyProvider interface { + ResponseBody() []byte + } + var responseBody responseBodyProvider + if errors.As(err, &responseBody) && responseBody != nil && len(responseBody.ResponseBody()) > 0 { + return string(responseBody.ResponseBody()) + } + return err.Error() +} + func claudeErrorDetailFromText(status int, errText string) (string, string) { message := strings.TrimSpace(errText) if message == "" { @@ -455,6 +678,8 @@ func claudeErrorTypeFromStatus(status int) string { return "permission_error" case http.StatusNotFound: return "not_found_error" + case http.StatusConflict: + return "conflict_error" case http.StatusRequestEntityTooLarge: return "request_too_large" case http.StatusTooManyRequests: diff --git a/sdk/api/handlers/gemini/gemini_handlers.go b/sdk/api/handlers/gemini/gemini_handlers.go index c643a52c3..4226cc9d0 100644 --- a/sdk/api/handlers/gemini/gemini_handlers.go +++ b/sdk/api/handlers/gemini/gemini_handlers.go @@ -13,6 +13,7 @@ import ( "time" "github.com/gin-gonic/gin" + "github.com/router-for-me/CLIProxyAPI/v8/internal/clienterror" . "github.com/router-for-me/CLIProxyAPI/v8/internal/constant" "github.com/router-for-me/CLIProxyAPI/v8/internal/interfaces" "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" @@ -151,7 +152,16 @@ func (h *GeminiAPIHandler) GeminiHandler(c *gin.Context) { } method := action[1] - rawJSON, _ := c.GetRawData() + rawJSON, errRead := c.GetRawData() + if errRead != nil && c.Request != nil && c.Request.Body != nil { + c.JSON(clienterror.HTTPStatusFromErrorOr(errRead, http.StatusBadRequest), handlers.ErrorResponse{ + Error: handlers.ErrorDetail{ + Message: fmt.Sprintf("Invalid request: %v", errRead), + Type: "invalid_request_error", + }, + }) + return + } switch method { case "generateContent": @@ -188,14 +198,44 @@ func (h *GeminiAPIHandler) handleStreamGenerateContent(c *gin.Context, modelName } cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) - dataChan, upstreamHeaders, errChan := h.ExecuteStreamWithAuthManager(cliCtx, h.HandlerType(), modelName, rawJSON, alt) - setSSEHeaders := func() { c.Header("Content-Type", "text/event-stream") c.Header("Cache-Control", "no-cache") c.Header("Connection", "keep-alive") c.Header("Access-Control-Allow-Origin", "*") } + var upstreamHeaders http.Header + committed := false + commitHeaders := func() { + if committed { + return + } + if alt == "" { + setSSEHeaders() + } + handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + committed = true + } + writeKeepAlive := func() bool { + commitHeaders() + _, _ = c.Writer.Write([]byte(": keep-alive\n\n")) + flusher.Flush() + return true + } + var keepAliveC <-chan time.Time + stopKeepAlive := func() {} + if alt == "" { + keepAliveC, stopKeepAlive = handlers.NewPreFirstTokenKeepAliveTicker(h.CurrentConfig()) + } + defer stopKeepAlive() + + dataChan, resolvedHeaders, errChan, okBootstrap := h.ExecuteStreamWithPreFTKeepAlive( + c, cliCtx, func(err error) { cliCancel(err) }, h.HandlerType(), modelName, rawJSON, alt, keepAliveC, writeKeepAlive, + ) + if !okBootstrap { + return + } + upstreamHeaders = resolvedHeaders // Peek at the first chunk for { @@ -209,6 +249,14 @@ func (h *GeminiAPIHandler) handleStreamGenerateContent(c *gin.Context, modelName errChan = nil continue } + if committed { + stopKeepAlive() + pendingErrors := make(chan *interfaces.ErrorMessage, 1) + pendingErrors <- errMsg + close(pendingErrors) + h.forwardGeminiStream(c, flusher, alt, func(err error) { cliCancel(err) }, make(chan []byte), pendingErrors) + return + } // Upstream failed immediately. Return proper error status and JSON. h.WriteErrorResponse(c, errMsg) if errMsg != nil { @@ -219,7 +267,15 @@ func (h *GeminiAPIHandler) handleStreamGenerateContent(c *gin.Context, modelName return case chunk, ok := <-dataChan: if !ok { + stopKeepAlive() if errMsg, hasPendingError := handlers.PendingStreamError(errChan); hasPendingError { + if committed { + pendingErrors := make(chan *interfaces.ErrorMessage, 1) + pendingErrors <- errMsg + close(pendingErrors) + h.forwardGeminiStream(c, flusher, alt, func(err error) { cliCancel(err) }, make(chan []byte), pendingErrors) + return + } h.WriteErrorResponse(c, errMsg) if errMsg != nil { cliCancel(errMsg.Error) @@ -229,20 +285,15 @@ func (h *GeminiAPIHandler) handleStreamGenerateContent(c *gin.Context, modelName return } // Closed without data - if alt == "" { - setSSEHeaders() - } - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + commitHeaders() flusher.Flush() cliCancel(nil) return } // Success! Set headers. - if alt == "" { - setSSEHeaders() - } - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + stopKeepAlive() + commitHeaders() // Write first chunk if alt == "" { @@ -257,6 +308,8 @@ func (h *GeminiAPIHandler) handleStreamGenerateContent(c *gin.Context, modelName // Continue h.forwardGeminiStream(c, flusher, alt, func(err error) { cliCancel(err) }, dataChan, errChan) return + case <-keepAliveC: + writeKeepAlive() } } } diff --git a/sdk/api/handlers/gemini/interactions_handlers.go b/sdk/api/handlers/gemini/interactions_handlers.go index 8e9264783..080b3a58f 100644 --- a/sdk/api/handlers/gemini/interactions_handlers.go +++ b/sdk/api/handlers/gemini/interactions_handlers.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/gin-gonic/gin" + "github.com/router-for-me/CLIProxyAPI/v8/internal/clienterror" . "github.com/router-for-me/CLIProxyAPI/v8/internal/constant" "github.com/router-for-me/CLIProxyAPI/v8/internal/interfaces" "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers" @@ -93,7 +94,7 @@ func buildInteractionsExecutionRequest(target interactionsRequestTarget, modelNa func (h *GeminiAPIHandler) Interactions(c *gin.Context) { rawJSON, errRead := c.GetRawData() if errRead != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{Error: handlers.ErrorDetail{Message: errRead.Error(), Type: "invalid_request_error"}}) + c.JSON(clienterror.HTTPStatusFromErrorOr(errRead, http.StatusBadRequest), handlers.ErrorResponse{Error: handlers.ErrorDetail{Message: errRead.Error(), Type: "invalid_request_error"}}) return } target, errParse := parseInteractionsRequestTarget(rawJSON) @@ -137,17 +138,67 @@ func (h *GeminiAPIHandler) handleInteractionsStream(c *gin.Context, cliCtx conte c.JSON(http.StatusInternalServerError, handlers.ErrorResponse{Error: handlers.ErrorDetail{Message: "Streaming not supported", Type: "server_error"}}) return } - stream, errMsg := h.ExecuteProtocolStreamWithAuthManager(cliCtx, req) + var upstreamHeaders http.Header + committed := false + commitSSE := func() { + if committed { + return + } + c.Header("Content-Type", "text/event-stream") + c.Header("Cache-Control", "no-cache") + c.Header("Connection", "keep-alive") + c.Header("Access-Control-Allow-Origin", "*") + handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + committed = true + } + keepAliveC, stopKeepAlive := handlers.NewPreFirstTokenKeepAliveTicker(h.CurrentConfig()) + defer stopKeepAlive() + + var stream handlers.ModelExecutionStream + var errMsg *interfaces.ErrorMessage + if keepAliveC == nil { + stream, errMsg = h.ExecuteProtocolStreamWithAuthManager(cliCtx, req) + } else { + type protocolStreamResult struct { + stream handlers.ModelExecutionStream + errMsg *interfaces.ErrorMessage + } + resultChan := make(chan protocolStreamResult, 1) + go func() { + s, e := h.ExecuteProtocolStreamWithAuthManager(cliCtx, req) + resultChan <- protocolStreamResult{stream: s, errMsg: e} + }() + for waiting := true; waiting; { + select { + case <-c.Request.Context().Done(): + cliCancel(c.Request.Context().Err()) + <-resultChan + return + case res := <-resultChan: + stream, errMsg = res.stream, res.errMsg + waiting = false + case <-keepAliveC: + commitSSE() + _, _ = c.Writer.Write([]byte(": keep-alive\n\n")) + flusher.Flush() + } + } + } + stopKeepAlive() if errMsg != nil { + if committed { + pendingErrors := make(chan *interfaces.ErrorMessage, 1) + pendingErrors <- errMsg + close(pendingErrors) + h.forwardInteractionsStream(c, flusher, func(err error) { cliCancel(err) }, make(chan []byte), pendingErrors) + return + } h.WriteErrorResponse(c, errMsg) cliCancel(errMsg.Error) return } - c.Header("Content-Type", "text/event-stream") - c.Header("Cache-Control", "no-cache") - c.Header("Connection", "keep-alive") - c.Header("Access-Control-Allow-Origin", "*") - handlers.WriteUpstreamHeaders(c.Writer.Header(), stream.Headers) + upstreamHeaders = stream.Headers + commitSSE() data := make(chan []byte) errs := make(chan *interfaces.ErrorMessage, 1) go func() { diff --git a/sdk/api/handlers/handlers.go b/sdk/api/handlers/handlers.go index 3c8aad929..be576f706 100644 --- a/sdk/api/handlers/handlers.go +++ b/sdk/api/handlers/handlers.go @@ -25,6 +25,7 @@ import ( coresession "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/session" coreusage "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/usage" "github.com/router-for-me/CLIProxyAPI/v8/sdk/config" + "github.com/router-for-me/CLIProxyAPI/v8/sdk/httptransport" "github.com/tidwall/gjson" "golang.org/x/net/context" ) @@ -63,7 +64,8 @@ const ( ) // BuildErrorResponseBody builds an OpenAI-compatible JSON error response body. -// If errText is already valid JSON, it is returned as-is to preserve upstream error payloads. +// If errText is already valid JSON, it is compacted to ensure single-line SSE compatibility +// while preserving upstream error payloads. func BuildErrorResponseBody(status int, errText string) []byte { return BuildErrorResponseBodyWithError(status, errText, nil) } @@ -111,6 +113,10 @@ func BuildErrorResponseBodyWithError(status int, errText string, err error) []by } if trimmed != "" && json.Valid([]byte(trimmed)) { + var compact bytes.Buffer + if errCompact := json.Compact(&compact, []byte(trimmed)); errCompact == nil { + return compact.Bytes() + } return []byte(trimmed) } @@ -152,6 +158,11 @@ func BuildErrorResponseBodyWithError(status int, errText string, err error) []by return payload } +var streamingKeepAliveTickerFactory = func(interval time.Duration) (<-chan time.Time, func()) { + ticker := time.NewTicker(interval) + return ticker.C, ticker.Stop +} + // StreamingKeepAliveInterval returns the streaming keep-alive interval for this server (SSE heartbeats and WebSocket Ping frames). // Returning 0 disables keep-alives (default when unset). func StreamingKeepAliveInterval(cfg *config.SDKConfig) time.Duration { @@ -165,6 +176,42 @@ func StreamingKeepAliveInterval(cfg *config.SDKConfig) time.Duration { return time.Duration(seconds) * time.Second } +// StreamingSendKeepAliveBeforeFT returns whether SSE keep-alives should be sent before the first token arrives. +// It is active only when streaming.send-keepalive-before-ft is true and streaming.keepalive-seconds > 0. +func StreamingSendKeepAliveBeforeFT(cfg *config.SDKConfig) bool { + return cfg != nil && cfg.Streaming.SendKeepAliveBeforeFT && StreamingKeepAliveInterval(cfg) > 0 +} + +// NewStreamingKeepAliveTicker returns a ticker channel and a stop function for streaming keep-alives. +// When interval <= 0, it returns a nil channel and a no-op stop function. +func NewStreamingKeepAliveTicker(interval time.Duration) (<-chan time.Time, func()) { + if interval <= 0 { + return nil, func() {} + } + return streamingKeepAliveTickerFactory(interval) +} + +// NewPreFirstTokenKeepAliveTicker returns a ticker channel and a stop function for pre-first-token SSE keep-alives. +// It is active only when streaming.send-keepalive-before-ft is true and streaming.keepalive-seconds > 0. +func NewPreFirstTokenKeepAliveTicker(cfg *config.SDKConfig) (<-chan time.Time, func()) { + if !StreamingSendKeepAliveBeforeFT(cfg) { + return nil, func() {} + } + return NewStreamingKeepAliveTicker(StreamingKeepAliveInterval(cfg)) +} + +// SetStreamingKeepAliveTickerFactoryForTest overrides the keep-alive ticker factory for deterministic tests +// and returns a restore function. +func SetStreamingKeepAliveTickerFactoryForTest(factory func(interval time.Duration) (<-chan time.Time, func())) func() { + prev := streamingKeepAliveTickerFactory + if factory != nil { + streamingKeepAliveTickerFactory = factory + } + return func() { + streamingKeepAliveTickerFactory = prev + } +} + // NonStreamingKeepAliveInterval returns the keep-alive interval for non-streaming responses. // Returning 0 disables keep-alives (default when unset). func NonStreamingKeepAliveInterval(cfg *config.SDKConfig) time.Duration { @@ -307,6 +354,28 @@ func requestCallerScope(ginCtx *gin.Context) string { return coresession.CallerScope(fmt.Sprint(value)) } +func addCallerScopeMetadata(meta map[string]any, callerScope string) { + if meta == nil { + return + } + callerScope = strings.TrimSpace(callerScope) + if callerScope == "" { + return + } + meta[coreexecutor.CallerScopeMetadataKey] = callerScope +} + +func callerScopeFromMetadata(meta map[string]any) string { + if len(meta) == 0 { + return "" + } + value, ok := meta[coreexecutor.CallerScopeMetadataKey].(string) + if !ok { + return "" + } + return strings.TrimSpace(value) +} + func addAuthSelectionModelMetadata(meta map[string]any, model string) { if meta == nil { return @@ -361,6 +430,8 @@ func setGenerateMetadata(meta map[string]any, rawJSON []byte) { // It holds a pool of clients to interact with the backend service and manages // load balancing, client selection, and configuration. type BaseAPIHandler struct { + runtimeMu sync.RWMutex + // AuthManager manages auth lifecycle and execution in the new architecture. AuthManager *coreauth.Manager @@ -397,7 +468,25 @@ func NewBaseAPIHandlers(cfg *config.SDKConfig, authManager *coreauth.Manager) *B // Parameters: // - clients: The new slice of AI service clients // - cfg: The new application configuration -func (h *BaseAPIHandler) UpdateClients(cfg *config.SDKConfig) { h.Cfg = cfg } +func (h *BaseAPIHandler) UpdateClients(cfg *config.SDKConfig) { + if h == nil { + return + } + h.runtimeMu.Lock() + h.Cfg = cfg + h.runtimeMu.Unlock() +} + +// CurrentConfig returns the current application configuration snapshot. +func (h *BaseAPIHandler) CurrentConfig() *config.SDKConfig { + if h == nil { + return nil + } + h.runtimeMu.RLock() + cfg := h.Cfg + h.runtimeMu.RUnlock() + return cfg +} // SetPluginHost configures the optional plugin interceptor host. func (h *BaseAPIHandler) SetPluginHost(host PluginInterceptorHost) { @@ -405,10 +494,11 @@ func (h *BaseAPIHandler) SetPluginHost(host PluginInterceptorHost) { return } if isNilPluginInterceptorHost(host) { - h.PluginHost = nil - return + host = nil } + h.runtimeMu.Lock() h.PluginHost = host + h.runtimeMu.Unlock() } // SetModelRouterHost configures the optional plugin model router host. @@ -417,10 +507,25 @@ func (h *BaseAPIHandler) SetModelRouterHost(host PluginModelRouterHost) { return } if isNilPluginModelRouterHost(host) { - h.ModelRouterHost = nil - return + host = nil } + h.runtimeMu.Lock() h.ModelRouterHost = host + h.runtimeMu.Unlock() +} + +// ModelRouterHostSnapshot returns the configured plugin model router host snapshot. +func (h *BaseAPIHandler) ModelRouterHostSnapshot() PluginModelRouterHost { + if h == nil { + return nil + } + h.runtimeMu.RLock() + host := h.ModelRouterHost + h.runtimeMu.RUnlock() + if isNilPluginModelRouterHost(host) { + return nil + } + return host } func isNilPluginInterceptorHost(host PluginInterceptorHost) bool { @@ -489,6 +594,9 @@ func (h *BaseAPIHandler) GetContextWithCancel(handler interfaces.APIHandler, c * requestCtx = c.Request.Context() } + if requestCtx != nil { + parentCtx = httptransport.InheritDecorator(parentCtx, requestCtx) + } if requestCtx != nil && logging.GetRequestID(parentCtx) == "" { if requestID := logging.GetRequestID(requestCtx); requestID != "" { parentCtx = logging.WithRequestID(parentCtx, requestID) @@ -546,7 +654,7 @@ func (h *BaseAPIHandler) GetContextWithCancel(handler interfaces.APIHandler, c * if c != nil { logging.SetResponseStatus(cancelCtx, c.Writer.Status()) } - if h.Cfg.RequestLog && len(params) == 1 { + if cfg := h.CurrentConfig(); cfg != nil && cfg.RequestLog && len(params) == 1 { if captured, exists := c.Get(logging.APIResponseCapturedContextKey); exists { if capturedBool, ok := captured.(bool); ok && capturedBool { cancel() @@ -598,7 +706,7 @@ func (h *BaseAPIHandler) StartNonStreamingKeepAlive(c *gin.Context, ctx context. if h == nil || c == nil { return func() {} } - interval := NonStreamingKeepAliveInterval(h.Cfg) + interval := NonStreamingKeepAliveInterval(h.CurrentConfig()) if interval <= 0 { return func() {} } diff --git a/sdk/api/handlers/handlers_errors.go b/sdk/api/handlers/handlers_errors.go index a35f2110d..7893da08e 100644 --- a/sdk/api/handlers/handlers_errors.go +++ b/sdk/api/handlers/handlers_errors.go @@ -128,7 +128,7 @@ func (h *BaseAPIHandler) WriteErrorResponse(c *gin.Context, msg *interfaces.Erro c.Writer.Header().Add("Retry-After", value) } } - if msg != nil && msg.Addon != nil && PassthroughHeadersEnabled(h.Cfg) { + if msg != nil && msg.Addon != nil && PassthroughHeadersEnabled(h.CurrentConfig()) { for key, values := range msg.Addon { if len(values) == 0 || IsCPAReservedResponseHeader(key) { continue @@ -196,7 +196,7 @@ func writeDirectErrorResponse(c *gin.Context, status int, msg *interfaces.ErrorM } func (h *BaseAPIHandler) LoggingAPIResponseError(ctx context.Context, err *interfaces.ErrorMessage) { - if h.Cfg.RequestLog { + if cfg := h.CurrentConfig(); cfg != nil && cfg.RequestLog { if ginContext, ok := ctx.Value("gin").(*gin.Context); ok { if apiResponseErrors, isExist := ginContext.Get("API_RESPONSE_ERROR"); isExist { if slicesAPIResponseError, isOk := apiResponseErrors.([]*interfaces.ErrorMessage); isOk { diff --git a/sdk/api/handlers/handlers_execution.go b/sdk/api/handlers/handlers_execution.go index 75eab384c..0f821c3e4 100644 --- a/sdk/api/handlers/handlers_execution.go +++ b/sdk/api/handlers/handlers_execution.go @@ -46,6 +46,8 @@ func (h *BaseAPIHandler) executeWithAuthManager(ctx context.Context, handlerType func (h *BaseAPIHandler) executeWithAuthManagerFormats(ctx context.Context, entryProtocol, exitProtocol, modelName string, rawJSON []byte, alt string, allowImageModel bool, execOptions modelExecutionOptions) ([]byte, http.Header, *interfaces.ErrorMessage) { originalRequestedModel := modelName + // Speech-only models are reachable solely through the speech entry protocol. + execOptions.AllowSpeechModel = isModelExecutionSpeechProtocol(entryProtocol) routeDecision := h.applyModelRouter(ctx, entryProtocol, modelName, rawJSON, false, execOptions) responseProtocol := modelExecutionResponseProtocol(entryProtocol, exitProtocol) if errMsg := validateNativeInteractionsExecution(entryProtocol, execOptions, routeDecision); errMsg != nil { @@ -63,6 +65,7 @@ func (h *BaseAPIHandler) executeWithAuthManagerFormats(ctx context.Context, entr if execOptions.Path != "" { reqMeta[coreexecutor.RequestPathMetadataKey] = execOptions.Path } + addCallerScopeMetadata(reqMeta, execOptions.CallerScope) reqMeta[coreexecutor.RequestedModelMetadataKey] = originalRequestedModel addAuthSelectionModelMetadata(reqMeta, execOptions.AuthSelectionModel) addModelExecutionSourceMetadata(reqMeta, execOptions.InternalSource) @@ -110,7 +113,7 @@ func (h *BaseAPIHandler) executeWithAuthManagerFormats(ctx context.Context, entr executedReq, executedOpts := afterAuthCapture.apply(req, opts) ctx = enrichContextWithSessionHierarchy(ctx, executedOpts.Headers, executedReq.Payload, executedOpts.Metadata) rawResponseHeaders := cloneHeader(resp.Headers) - passthroughHeaders := executionPassthroughHeaders(h.Cfg, execOptions.InternalSource) + passthroughHeaders := executionPassthroughHeaders(h.CurrentConfig(), execOptions.InternalSource) responseHeaders := downstreamHeadersFromExecutor(rawResponseHeaders, passthroughHeaders) body, responseHeaders := h.applyResponseInterceptors(ctx, lifecycle.requestID(), responseProtocol, normalizedModel, originalRequestedModel, executedOpts, rawResponseHeaders, responseHeaders, executedOpts.OriginalRequest, executedReq.Payload, resp.Payload, http.StatusOK, execOptions.SkipInterceptorPluginID, passthroughHeaders) lifecycle.complete(pluginapi.RequestCompletionSucceeded, http.StatusOK, nil) @@ -135,6 +138,7 @@ func (h *BaseAPIHandler) executeCountWithAuthManager(ctx context.Context, handle } providers = adjustExecutionProvidersForEntryProtocol(handlerType, providers) reqMeta := requestExecutionMetadata(ctx) + addCallerScopeMetadata(reqMeta, execOptions.CallerScope) reqMeta[coreexecutor.RequestedModelMetadataKey] = originalRequestedModel addAuthSelectionModelMetadata(reqMeta, execOptions.AuthSelectionModel) setReasoningEffortMetadata(reqMeta, handlerType, normalizedModel, rawJSON) @@ -180,7 +184,7 @@ func (h *BaseAPIHandler) executeCountWithAuthManager(ctx context.Context, handle executedReq, executedOpts := afterAuthCapture.apply(req, opts) ctx = enrichContextWithSessionHierarchy(ctx, executedOpts.Headers, executedReq.Payload, executedOpts.Metadata) rawResponseHeaders := cloneHeader(resp.Headers) - passthroughHeaders := executionPassthroughHeaders(h.Cfg, execOptions.InternalSource) + passthroughHeaders := executionPassthroughHeaders(h.CurrentConfig(), execOptions.InternalSource) responseHeaders := downstreamHeadersFromExecutor(rawResponseHeaders, passthroughHeaders) body, responseHeaders := h.applyResponseInterceptors(ctx, lifecycle.requestID(), handlerType, normalizedModel, originalRequestedModel, executedOpts, rawResponseHeaders, responseHeaders, executedOpts.OriginalRequest, executedReq.Payload, resp.Payload, http.StatusOK, execOptions.SkipInterceptorPluginID, passthroughHeaders) lifecycle.complete(pluginapi.RequestCompletionSucceeded, http.StatusOK, nil) @@ -230,7 +234,7 @@ func (h *BaseAPIHandler) executeWithPluginExecutor(ctx context.Context, entryPro reporter.EnsurePublished(execCtx) } rawResponseHeaders := cloneHeader(resp.Headers) - passthroughHeaders := executionPassthroughHeaders(h.Cfg, execOptions.InternalSource) + passthroughHeaders := executionPassthroughHeaders(h.CurrentConfig(), execOptions.InternalSource) responseHeaders := downstreamHeadersFromExecutor(rawResponseHeaders, passthroughHeaders) body, responseHeaders := h.applyResponseInterceptors(execCtx, lifecycle.requestID(), responseProtocol, modelName, originalRequestedModel, opts, rawResponseHeaders, responseHeaders, opts.OriginalRequest, req.Payload, resp.Payload, http.StatusOK, execOptions.SkipInterceptorPluginID, passthroughHeaders) lifecycle.complete(pluginapi.RequestCompletionSucceeded, http.StatusOK, nil) @@ -266,7 +270,7 @@ func (h *BaseAPIHandler) countWithPluginExecutor(ctx context.Context, handlerTyp return nil, nil, errMsg } rawResponseHeaders := cloneHeader(resp.Headers) - passthroughHeaders := executionPassthroughHeaders(h.Cfg, execOptions.InternalSource) + passthroughHeaders := executionPassthroughHeaders(h.CurrentConfig(), execOptions.InternalSource) responseHeaders := downstreamHeadersFromExecutor(rawResponseHeaders, passthroughHeaders) body, responseHeaders := h.applyResponseInterceptors(ctx, lifecycle.requestID(), handlerType, modelName, originalRequestedModel, opts, rawResponseHeaders, responseHeaders, opts.OriginalRequest, req.Payload, resp.Payload, http.StatusOK, execOptions.SkipInterceptorPluginID, passthroughHeaders) lifecycle.complete(pluginapi.RequestCompletionSucceeded, http.StatusOK, nil) @@ -278,6 +282,7 @@ func (h *BaseAPIHandler) pluginExecutorRequest(ctx context.Context, entryProtoco if execOptions.Path != "" { reqMeta[coreexecutor.RequestPathMetadataKey] = execOptions.Path } + addCallerScopeMetadata(reqMeta, execOptions.CallerScope) reqMeta[coreexecutor.RequestedModelMetadataKey] = originalRequestedModel addAuthSelectionModelMetadata(reqMeta, execOptions.AuthSelectionModel) addModelExecutionSourceMetadata(reqMeta, execOptions.InternalSource) @@ -398,11 +403,19 @@ func (h *BaseAPIHandler) pluginExecutorHost() PluginExecutorHost { if h == nil { return nil } - if executorHost, ok := h.ModelRouterHost.(PluginExecutorHost); ok && executorHost != nil { - return executorHost + h.runtimeMu.RLock() + routerHost := h.ModelRouterHost + pluginHost := h.PluginHost + h.runtimeMu.RUnlock() + if !isNilPluginModelRouterHost(routerHost) { + if executorHost, ok := routerHost.(PluginExecutorHost); ok && executorHost != nil { + return executorHost + } } - if executorHost, ok := h.PluginHost.(PluginExecutorHost); ok && executorHost != nil { - return executorHost + if !isNilPluginInterceptorHost(pluginHost) { + if executorHost, ok := pluginHost.(PluginExecutorHost); ok && executorHost != nil { + return executorHost + } } return nil } diff --git a/sdk/api/handlers/handlers_interceptors.go b/sdk/api/handlers/handlers_interceptors.go index a8acb2a95..330ab5132 100644 --- a/sdk/api/handlers/handlers_interceptors.go +++ b/sdk/api/handlers/handlers_interceptors.go @@ -245,6 +245,11 @@ func nextStreamChunk(ctx context.Context, pending *[]coreexecutor.StreamChunk, c case <-ctx.Done(): return coreexecutor.StreamChunk{}, false, true case chunk, ok = <-chunks: + // A channel closed together with the cancellation was cut by it: the select above picks at random when both are + // ready, and the plain close would be reported as a success. + if !ok && ctx.Err() != nil { + return coreexecutor.StreamChunk{}, false, true + } } } else { chunk, ok = <-chunks @@ -340,7 +345,13 @@ func (h *BaseAPIHandler) interceptorHost() PluginInterceptorHost { if h == nil { return nil } - return h.PluginHost + h.runtimeMu.RLock() + host := h.PluginHost + h.runtimeMu.RUnlock() + if isNilPluginInterceptorHost(host) { + return nil + } + return host } func streamInterceptorsEnabled(host PluginInterceptorHost) bool { @@ -493,6 +504,7 @@ func (h *BaseAPIHandler) applyRequestInterceptorsBeforeAuth(ctx context.Context, Stream: opts.Stream, Headers: cloneHeader(opts.Headers), Body: req.Payload, + CallerScope: callerScopeFromMetadata(opts.Metadata), Metadata: opts.Metadata, }, skipPluginID) opts.Headers = finalInterceptorHeaders(opts.Headers, resp.Headers) @@ -583,6 +595,7 @@ func (h *BaseAPIHandler) applyRequestInterceptorsAfterAuth(ctx context.Context, Stream: req.Stream, Headers: cloneHeader(req.Headers), Body: req.Body, + CallerScope: callerScopeFromMetadata(req.Metadata), Metadata: req.Metadata, }, skipPluginID) return coreexecutor.RequestAfterAuthInterceptResponse{ @@ -623,6 +636,9 @@ func (h *BaseAPIHandler) applyResponseInterceptors(ctx context.Context, requestI return body, responseHeaders } +// ModelDetailIDContextKey requests selection from the final, plugin-filtered catalog. +const ModelDetailIDContextKey = "cliproxy.model_detail_id" + // WriteModelListResponse serializes the model-list payload, applies plugin response interceptors // if a plugin host is configured, and writes the resulting headers and body to the Gin context. func (h *BaseAPIHandler) WriteModelListResponse(c *gin.Context, sourceFormat string, payload any) { @@ -646,6 +662,7 @@ func (h *BaseAPIHandler) WriteModelListResponse(c *gin.Context, sourceFormat str "Content-Type": []string{"application/json; charset=utf-8"}, } + var lifecycle *requestLifecycleTracker host := h.interceptorHost() if host != nil { ctx := context.Background() @@ -656,7 +673,7 @@ func (h *BaseAPIHandler) WriteModelListResponse(c *gin.Context, sourceFormat str ctx = reqCtx } } - lifecycle := h.newRequestLifecycleTracker(ctx, sourceFormat, "", "", false, nil, "") + lifecycle = h.newRequestLifecycleTracker(ctx, sourceFormat, "", "", false, nil, "") resp := interceptResponse(ctx, host, pluginapi.ResponseInterceptRequest{ RequestID: lifecycle.requestID(), SourceFormat: sourceFormat, @@ -679,9 +696,44 @@ func (h *BaseAPIHandler) WriteModelListResponse(c *gin.Context, sourceFormat str c.Writer.Header()[key] = append([]string(nil), values...) } } - lifecycle.complete(pluginapi.RequestCompletionSucceeded, http.StatusOK, nil) } + if modelID, detail := c.Get(ModelDetailIDContextKey); detail { + c.Writer.Header().Del("Content-Length") + var catalog struct { + Data []json.RawMessage `json:"data"` + Models []json.RawMessage `json:"models"` + } + if errUnmarshal := json.Unmarshal(body, &catalog); errUnmarshal != nil { + lifecycle.complete(pluginapi.RequestCompletionFailed, http.StatusBadGateway, errUnmarshal) + c.JSON(http.StatusBadGateway, gin.H{"error": gin.H{"message": "Invalid model catalog", "type": "api_error"}}) + return + } + var selected json.RawMessage + for _, model := range append(catalog.Data, catalog.Models...) { + var entry struct { + ID string `json:"id"` + Slug string `json:"slug"` + } + if errUnmarshal := json.Unmarshal(model, &entry); errUnmarshal != nil { + continue + } + if entry.ID == "" { + entry.ID = entry.Slug + } + if entry.ID != "" && entry.ID == modelID { + selected = model + break + } + } + if selected == nil { + lifecycle.complete(pluginapi.RequestCompletionFailed, http.StatusNotFound, nil) + c.JSON(http.StatusNotFound, gin.H{"error": gin.H{"message": "Model not found", "type": "invalid_request_error", "code": "model_not_found"}}) + return + } + body = selected + } + lifecycle.complete(pluginapi.RequestCompletionSucceeded, http.StatusOK, nil) if c.Writer.Header().Get("Content-Type") == "" { c.Header("Content-Type", "application/json; charset=utf-8") } diff --git a/sdk/api/handlers/handlers_routing.go b/sdk/api/handlers/handlers_routing.go index e7297704c..e4e03f017 100644 --- a/sdk/api/handlers/handlers_routing.go +++ b/sdk/api/handlers/handlers_routing.go @@ -98,7 +98,7 @@ func excludeExecutionProvider(providers []string, excluded string) []string { } func (h *BaseAPIHandler) getRequestDetails(modelName string) (providers []string, normalizedModel string, err *interfaces.ErrorMessage) { - return h.getRequestDetailsWithOptions(modelName, false) + return h.getRequestDetailsWithOptions(modelName, false, false) } func validateNativeInteractionsExecution(entryProtocol string, execOptions modelExecutionOptions, routeDecision modelRouteDecision) *interfaces.ErrorMessage { @@ -141,6 +141,9 @@ func (h *BaseAPIHandler) providersForExecution(modelName, originalRequestedModel if errMsg := h.validateImageOnlyModel(normalizedModel, allowImageModel); errMsg != nil { return nil, "", errMsg } + if errMsg := h.validateSpeechOnlyModel(normalizedModel, execOptions.AllowSpeechModel); errMsg != nil { + return nil, "", errMsg + } return []string{forcedProvider}, normalizedModel, nil } if routeDecision.Provider != "" { @@ -151,12 +154,15 @@ func (h *BaseAPIHandler) providersForExecution(modelName, originalRequestedModel if errMsg := h.validateImageOnlyModel(normalizedModel, allowImageModel); errMsg != nil { return nil, "", errMsg } + if errMsg := h.validateSpeechOnlyModel(normalizedModel, execOptions.AllowSpeechModel); errMsg != nil { + return nil, "", errMsg + } return []string{routeDecision.Provider}, normalizedModel, nil } - return h.getRequestDetailsWithOptions(modelName, allowImageModel) + return h.getRequestDetailsWithOptions(modelName, allowImageModel, execOptions.AllowSpeechModel) } -func (h *BaseAPIHandler) getRequestDetailsWithOptions(modelName string, allowImageModel bool) (providers []string, normalizedModel string, err *interfaces.ErrorMessage) { +func (h *BaseAPIHandler) getRequestDetailsWithOptions(modelName string, allowImageModel, allowSpeechModel bool) (providers []string, normalizedModel string, err *interfaces.ErrorMessage) { resolvedModelName := modelName initialSuffix := thinking.ParseSuffix(modelName) if initialSuffix.ModelName == "auto" { @@ -184,6 +190,9 @@ func (h *BaseAPIHandler) getRequestDetailsWithOptions(modelName string, allowIma if errMsg := h.validateImageOnlyModel(baseModel, allowImageModel); errMsg != nil { return nil, "", errMsg } + if errMsg := h.validateSpeechOnlyModel(baseModel, allowSpeechModel); errMsg != nil { + return nil, "", errMsg + } if h != nil && h.AuthManager != nil && h.AuthManager.HomeEnabled() { return []string{"home"}, resolvedModelName, nil @@ -246,6 +255,31 @@ func isOpenAIImageOnlyModel(model string) bool { } } +// validateSpeechOnlyModel rejects speech-only models outside the speech endpoints so they are +// never sent to a chat upstream. +func (h *BaseAPIHandler) validateSpeechOnlyModel(modelName string, allowSpeechModel bool) *interfaces.ErrorMessage { + baseModel := strings.TrimSpace(thinking.ParseSuffix(modelName).ModelName) + if baseModel == "" { + baseModel = strings.TrimSpace(modelName) + } + if isXAISpeechOnlyModel(baseModel) && !allowSpeechModel { + return &interfaces.ErrorMessage{ + StatusCode: http.StatusBadRequest, + Error: fmt.Errorf("model %s is only supported on /v1/audio/speech and /v1/tts", routeModelBaseName(baseModel)), + } + } + return nil +} + +func isXAISpeechOnlyModel(model string) bool { + switch strings.ToLower(strings.TrimSpace(routeModelBaseName(model))) { + case "grok-tts", "grok-voice-tts-1.0": + return true + default: + return false + } +} + func routeModelBaseName(model string) string { model = strings.TrimSpace(model) if idx := strings.LastIndex(model, "/"); idx >= 0 && idx < len(model)-1 { @@ -267,14 +301,17 @@ func (h *BaseAPIHandler) modelRouterHost() PluginModelRouterHost { if h == nil { return nil } - if !isNilPluginModelRouterHost(h.ModelRouterHost) { - return h.ModelRouterHost + h.runtimeMu.RLock() + routerHost := h.ModelRouterHost + pluginHost := h.PluginHost + h.runtimeMu.RUnlock() + if !isNilPluginModelRouterHost(routerHost) { + return routerHost } - host := h.interceptorHost() - if host == nil { + if isNilPluginInterceptorHost(pluginHost) { return nil } - router, ok := host.(PluginModelRouterHost) + router, ok := pluginHost.(PluginModelRouterHost) if !ok { return nil } @@ -329,6 +366,7 @@ func (h *BaseAPIHandler) applyModelRouter(ctx context.Context, handlerType, mode return decision } meta := requestExecutionMetadata(ctx) + addCallerScopeMetadata(meta, execOptions.CallerScope) meta[coreexecutor.RequestedModelMetadataKey] = modelName addModelExecutionSourceMetadata(meta, execOptions.InternalSource) resp, ok := routeModel(ctx, host, pluginapi.ModelRouteRequest{ diff --git a/sdk/api/handlers/handlers_stream.go b/sdk/api/handlers/handlers_stream.go index 5ac61ec93..102a38532 100644 --- a/sdk/api/handlers/handlers_stream.go +++ b/sdk/api/handlers/handlers_stream.go @@ -3,6 +3,7 @@ package handlers import ( "bytes" "encoding/json" + "errors" "fmt" "net/http" @@ -91,7 +92,7 @@ func (h *BaseAPIHandler) streamWithPluginExecutor(ctx context.Context, entryProt return nil, nil, errChan } - passthroughHeadersEnabled := executionPassthroughHeaders(h.Cfg, execOptions.InternalSource) + passthroughHeadersEnabled := executionPassthroughHeaders(h.CurrentConfig(), execOptions.InternalSource) interceptorHost := h.interceptorHost() streamInterceptorsActive := streamInterceptorsEnabled(interceptorHost) rawStreamHeaders := cloneHeader(streamResult.Headers) @@ -150,6 +151,7 @@ func (h *BaseAPIHandler) streamWithPluginExecutor(ctx context.Context, entryProt var completionErr error var streamUsage helps.StreamUsageBuffer defer func() { + completionOutcome, completionStatus, completionErr = streamDeliveryCompletion(ctx, completionOutcome, completionStatus, completionErr) lifecycle.complete(completionOutcome, completionStatus, completionErr) if reporter != nil && !nestedTracker.hasNestedExecution() { if completionOutcome != pluginapi.RequestCompletionSucceeded && completionErr != nil { @@ -185,11 +187,6 @@ func (h *BaseAPIHandler) streamWithPluginExecutor(ctx context.Context, entryProt select { case errChan <- &interfaces.ErrorMessage{StatusCode: http.StatusBadGateway, Error: errValidate}: case <-done: - completionOutcome = pluginapi.RequestCompletionCanceled - completionStatus = 0 - if ctx != nil { - completionErr = ctx.Err() - } } } } @@ -203,11 +200,6 @@ func (h *BaseAPIHandler) streamWithPluginExecutor(ctx context.Context, entryProt select { case errChan <- errMsg: case <-done: - completionOutcome = pluginapi.RequestCompletionCanceled - completionStatus = 0 - if ctx != nil { - completionErr = ctx.Err() - } } return } @@ -259,11 +251,6 @@ func (h *BaseAPIHandler) streamWithPluginExecutor(ctx context.Context, entryProt select { case errChan <- &interfaces.ErrorMessage{StatusCode: http.StatusBadGateway, Error: errValidate}: case <-done: - completionOutcome = pluginapi.RequestCompletionCanceled - completionStatus = 0 - if ctx != nil { - completionErr = ctx.Err() - } } return } @@ -296,6 +283,8 @@ func (h *BaseAPIHandler) executeStreamWithAuthManager(ctx context.Context, handl func (h *BaseAPIHandler) executeStreamWithAuthManagerFormats(ctx context.Context, entryProtocol, exitProtocol, modelName string, rawJSON []byte, alt string, allowImageModel bool, execOptions modelExecutionOptions) (<-chan []byte, http.Header, <-chan *interfaces.ErrorMessage) { originalRequestedModel := modelName + // Speech-only models are reachable solely through the speech entry protocol. + execOptions.AllowSpeechModel = isModelExecutionSpeechProtocol(entryProtocol) routeDecision, preparedRoute := preparedModelRouteFromContext(ctx, execOptions.SkipRouterPluginID) if !preparedRoute { routeDecision = h.applyModelRouter(ctx, entryProtocol, modelName, rawJSON, true, execOptions) @@ -322,6 +311,7 @@ func (h *BaseAPIHandler) executeStreamWithAuthManagerFormats(ctx context.Context if execOptions.Path != "" { reqMeta[coreexecutor.RequestPathMetadataKey] = execOptions.Path } + addCallerScopeMetadata(reqMeta, execOptions.CallerScope) reqMeta[coreexecutor.RequestedModelMetadataKey] = originalRequestedModel addAuthSelectionModelMetadata(reqMeta, execOptions.AuthSelectionModel) addModelExecutionSourceMetadata(reqMeta, execOptions.InternalSource) @@ -386,7 +376,7 @@ func (h *BaseAPIHandler) executeStreamWithAuthManagerFormats(ctx context.Context if executedReq, executedOpts := executedRequest(); len(executedOpts.Headers) > 0 || len(executedReq.Payload) > 0 || len(executedOpts.Metadata) > 0 { ctx = enrichContextWithSessionHierarchy(ctx, executedOpts.Headers, executedReq.Payload, executedOpts.Metadata) } - passthroughHeadersEnabled := executionPassthroughHeaders(h.Cfg, execOptions.InternalSource) + passthroughHeadersEnabled := executionPassthroughHeaders(h.CurrentConfig(), execOptions.InternalSource) interceptorHost := h.interceptorHost() streamInterceptorsActive := streamInterceptorsEnabled(interceptorHost) // Resolve bootstrap retries and header initialization before returning so the @@ -499,17 +489,10 @@ func (h *BaseAPIHandler) executeStreamWithAuthManagerFormats(ctx context.Context var bootstrapErr *interfaces.ErrorMessage readInitialStreamChunks := func() { for { - var chunk coreexecutor.StreamChunk - var ok bool - if ctx != nil { - select { - case <-ctx.Done(): - streamCanceledBeforeRead = true - return - case chunk, ok = <-chunks: - } - } else { - chunk, ok = <-chunks + chunk, ok, canceled := nextStreamChunk(ctx, nil, nil, chunks) + if canceled { + streamCanceledBeforeRead = true + return } if !ok { streamClosedBeforeRead = true @@ -550,7 +533,7 @@ func (h *BaseAPIHandler) executeStreamWithAuthManagerFormats(ctx context.Context } } - maxBootstrapRetries := StreamingBootstrapRetries(h.Cfg) + maxBootstrapRetries := StreamingBootstrapRetries(h.CurrentConfig()) if h.AuthManager.HomeEnabled() { maxBootstrapRetries = 0 } @@ -609,6 +592,7 @@ func (h *BaseAPIHandler) executeStreamWithAuthManagerFormats(ctx context.Context completionStatus := http.StatusOK var completionErr error defer func() { + completionOutcome, completionStatus, completionErr = streamDeliveryCompletion(ctx, completionOutcome, completionStatus, completionErr) lifecycle.complete(completionOutcome, completionStatus, completionErr) }() defer close(dataChan) @@ -655,11 +639,7 @@ func (h *BaseAPIHandler) executeStreamWithAuthManagerFormats(ctx context.Context } completionStatus = bootstrapErr.StatusCode completionErr = bootstrapErr.Error - if !sendErr(bootstrapErr) && ctx != nil && ctx.Err() != nil { - completionOutcome = pluginapi.RequestCompletionCanceled - completionStatus = 0 - completionErr = ctx.Err() - } + _ = sendErr(bootstrapErr) return } @@ -705,11 +685,7 @@ func (h *BaseAPIHandler) executeStreamWithAuthManagerFormats(ctx context.Context completionOutcome = pluginapi.RequestCompletionFailed completionStatus = errMsg.StatusCode completionErr = chunk.Err - if !sendErr(errMsg) && ctx != nil && ctx.Err() != nil { - completionOutcome = pluginapi.RequestCompletionCanceled - completionStatus = 0 - completionErr = ctx.Err() - } + _ = sendErr(errMsg) return } if len(chunk.Payload) == 0 { @@ -720,11 +696,7 @@ func (h *BaseAPIHandler) executeStreamWithAuthManagerFormats(ctx context.Context completionOutcome = pluginapi.RequestCompletionFailed completionStatus = errMsg.StatusCode completionErr = errMsg.Error - if !sendErr(errMsg) && ctx != nil && ctx.Err() != nil { - completionOutcome = pluginapi.RequestCompletionCanceled - completionStatus = 0 - completionErr = ctx.Err() - } + _ = sendErr(errMsg) return } if !deliverable { @@ -871,3 +843,22 @@ func validateSSEDataJSON(chunk []byte) error { } return state.Finish() } + +// streamDeliveryCompletion reconciles execution cleanup with the HTTP outcome. +// Call only after closing output channels: the consumer may need EOF to finish. +func streamDeliveryCompletion(ctx context.Context, outcome pluginapi.RequestCompletionOutcome, status int, err error) (pluginapi.RequestCompletionOutcome, int, error) { + if ctx == nil { + return outcome, status, err + } + deliveryErr, tracked := coreusage.WaitStreamDelivery(ctx) + if !tracked || (err != nil && !errors.Is(err, context.Canceled)) || outcome == pluginapi.RequestCompletionRejected { + return outcome, status, err + } + if deliveryErr == nil { + return pluginapi.RequestCompletionSucceeded, http.StatusOK, nil + } + if errors.Is(deliveryErr, context.Canceled) || errors.Is(deliveryErr, context.DeadlineExceeded) { + return pluginapi.RequestCompletionCanceled, 0, deliveryErr + } + return pluginapi.RequestCompletionFailed, executionErrorMessage(deliveryErr).StatusCode, deliveryErr +} diff --git a/sdk/api/handlers/model_execution.go b/sdk/api/handlers/model_execution.go index daf1f53e6..c98c26600 100644 --- a/sdk/api/handlers/model_execution.go +++ b/sdk/api/handlers/model_execution.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/router-for-me/CLIProxyAPI/v8/internal/interfaces" + "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/usage" "github.com/router-for-me/CLIProxyAPI/v8/sdk/proxyutil" "golang.org/x/net/context" ) @@ -26,6 +27,8 @@ type modelExecutionOptions struct { AuthSelectionModel string ProxyURL string Path string + CallerScope string + AllowSpeechModel bool } // ProtocolExecutionRequest describes a route-level model execution request with explicit protocols. @@ -59,6 +62,7 @@ type ModelExecutionRequest struct { AuthID string ProxyURL string Path string + CallerScope string } // ModelExecutionResponse describes a non-streaming internal model execution response. @@ -105,6 +109,7 @@ func (e *ModelExecutionStreamError) Error() string { // for the nested model execution while other plugins may still run. func (h *BaseAPIHandler) ExecuteModel(ctx context.Context, req ModelExecutionRequest) (ModelExecutionResponse, *interfaces.ErrorMessage) { markNestedExecution(ctx) + ctx = usage.WithoutStreamDelivery(ctx) if req.Stream { return ModelExecutionResponse{}, modelExecutionModeError("ExecuteModel requires Stream=false") } @@ -124,6 +129,7 @@ func (h *BaseAPIHandler) ExecuteModel(ctx context.Context, req ModelExecutionReq ForcedProvider: req.ForcedProvider, ProxyURL: strings.TrimSpace(req.ProxyURL), Path: strings.TrimSpace(req.Path), + CallerScope: strings.TrimSpace(req.CallerScope), }) if errMsg != nil { return ModelExecutionResponse{}, errMsg @@ -141,6 +147,7 @@ func (h *BaseAPIHandler) ExecuteModel(ctx context.Context, req ModelExecutionReq // for the nested model execution while other plugins may still run. func (h *BaseAPIHandler) ExecuteModelStream(ctx context.Context, req ModelExecutionRequest) (ModelExecutionStream, *interfaces.ErrorMessage) { markNestedExecution(ctx) + ctx = usage.WithoutStreamDelivery(ctx) if !req.Stream { return ModelExecutionStream{}, modelExecutionModeError("ExecuteModelStream requires Stream=true") } @@ -160,6 +167,7 @@ func (h *BaseAPIHandler) ExecuteModelStream(ctx context.Context, req ModelExecut ForcedProvider: req.ForcedProvider, ProxyURL: strings.TrimSpace(req.ProxyURL), Path: strings.TrimSpace(req.Path), + CallerScope: strings.TrimSpace(req.CallerScope), }) chunks, errMsg := prepareModelExecutionStream(ctx, dataChan, errChan) if errMsg != nil { @@ -227,6 +235,10 @@ func isModelExecutionImageProtocol(protocol string) bool { return strings.EqualFold(strings.TrimSpace(protocol), "openai-image") } +func isModelExecutionSpeechProtocol(protocol string) bool { + return strings.EqualFold(strings.TrimSpace(protocol), "openai-speech") +} + func validateModelExecutionProxy(raw string) *interfaces.ErrorMessage { raw = strings.TrimSpace(raw) if raw == "" { diff --git a/sdk/api/handlers/native_tool_search_capability.go b/sdk/api/handlers/native_tool_search_capability.go new file mode 100644 index 000000000..a685daa69 --- /dev/null +++ b/sdk/api/handlers/native_tool_search_capability.go @@ -0,0 +1,10 @@ +package handlers + +// SupportsNativeToolSearchModel reports whether every credential serving the +// public model can receive Codex tool_search as Anthropic native tool search. +func (h *BaseAPIHandler) SupportsNativeToolSearchModel(model string) bool { + if h == nil || h.AuthManager == nil { + return false + } + return h.AuthManager.SupportsNativeToolSearchForModel(model) +} diff --git a/sdk/api/handlers/openai/codex_client_models.go b/sdk/api/handlers/openai/codex_client_models.go index 57f9bce14..2b2e21859 100644 --- a/sdk/api/handlers/openai/codex_client_models.go +++ b/sdk/api/handlers/openai/codex_client_models.go @@ -3,6 +3,7 @@ package openai import ( codexmodels "github.com/router-for-me/CLIProxyAPI/v8/internal/client/codex/models" "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + responsesclaude "github.com/router-for-me/CLIProxyAPI/v8/internal/translator/claude/openai/responses" ) func (h *OpenAIAPIHandler) codexClientModelsResponse(clientVersion ...string) map[string]any { @@ -10,13 +11,22 @@ func (h *OpenAIAPIHandler) codexClientModelsResponse(clientVersion ...string) ma if len(clientVersion) > 0 { version = clientVersion[0] } - optimizeMultiAgentV2 := h != nil && h.Cfg != nil && h.Cfg.Client.Codex.OptimizeMultiAgentV2 + optimizeMultiAgentV2 := false var applyPatchCapabilityForModel codexmodels.ApplyPatchCapabilityForModelFunc - if h != nil && h.Cfg != nil && h.Cfg.Client.Codex.EnableApplyPatch { - applyPatchCapabilityForModel = h.SupportsApplyPatchModel + if h != nil { + if cfg := h.CurrentConfig(); cfg != nil { + optimizeMultiAgentV2 = cfg.Client.Codex.OptimizeMultiAgentV2 + if cfg.Client.Codex.EnableApplyPatch { + applyPatchCapabilityForModel = h.SupportsApplyPatchModel + } + } + } + var toolSearchCapabilityForModel codexmodels.ToolSearchCapabilityForModelFunc + if h != nil && responsesclaude.ClientToolSearchEnabled() { + toolSearchCapabilityForModel = h.SupportsNativeToolSearchModel } modelRegistry := registry.GetGlobalRegistry() - return codexmodels.BuildResponseForClientWithToolCapabilities(h.Models(), modelRegistry.GetModelProviders, modelRegistry.GetResponsesWebSearchCapability, applyPatchCapabilityForModel, optimizeMultiAgentV2, version) + return codexmodels.BuildResponseForClientWithToolCapabilities(h.Models(), modelRegistry.GetModelProviders, modelRegistry.GetResponsesWebSearchCapability, applyPatchCapabilityForModel, toolSearchCapabilityForModel, optimizeMultiAgentV2, version) } // CodexClientModelsResponse builds a Codex client model response. diff --git a/sdk/api/handlers/openai/openai_handlers.go b/sdk/api/handlers/openai/openai_handlers.go index 088e8fd82..e0e72458b 100644 --- a/sdk/api/handlers/openai/openai_handlers.go +++ b/sdk/api/handlers/openai/openai_handlers.go @@ -7,6 +7,7 @@ package openai import ( + "bytes" "context" "encoding/json" "fmt" @@ -15,6 +16,7 @@ import ( "github.com/gin-gonic/gin" codexmodels "github.com/router-for-me/CLIProxyAPI/v8/internal/client/codex/models" + "github.com/router-for-me/CLIProxyAPI/v8/internal/clienterror" . "github.com/router-for-me/CLIProxyAPI/v8/internal/constant" "github.com/router-for-me/CLIProxyAPI/v8/internal/interfaces" "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" @@ -116,7 +118,7 @@ func (h *OpenAIAPIHandler) ChatCompletions(c *gin.Context) { rawJSON, err := handlers.ReadRequestBody(c) // If data retrieval fails, return a 400 Bad Request error. if err != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ + c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), handlers.ErrorResponse{ Error: handlers.ErrorDetail{ Message: fmt.Sprintf("Invalid request: %v", err), Type: "invalid_request_error", @@ -133,7 +135,7 @@ func (h *OpenAIAPIHandler) ChatCompletions(c *gin.Context) { // Convert them to Chat Completions so downstream translators preserve tool metadata. if shouldTreatAsResponsesFormat(rawJSON) { modelName := gjson.GetBytes(rawJSON, "model").String() - rawJSON = responsesconverter.ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName, rawJSON, stream) + rawJSON, _ = responsesconverter.ConvertOpenAIResponsesRequestToOpenAIChatCompletions(modelName, rawJSON, stream) stream = gjson.GetBytes(rawJSON, "stream").Bool() } @@ -171,7 +173,7 @@ func (h *OpenAIAPIHandler) Completions(c *gin.Context) { rawJSON, err := handlers.ReadRequestBody(c) // If data retrieval fails, return a 400 Bad Request error. if err != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ + c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), handlers.ErrorResponse{ Error: handlers.ErrorDetail{ Message: fmt.Sprintf("Invalid request: %v", err), Type: "invalid_request_error", @@ -483,14 +485,38 @@ func (h *OpenAIAPIHandler) handleStreamingResponse(c *gin.Context, rawJSON []byt modelName := gjson.GetBytes(rawJSON, "model").String() cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) - dataChan, upstreamHeaders, errChan := h.ExecuteStreamWithAuthManager(cliCtx, h.HandlerType(), modelName, rawJSON, h.GetAlt(c)) - setSSEHeaders := func() { c.Header("Content-Type", "text/event-stream") c.Header("Cache-Control", "no-cache") c.Header("Connection", "keep-alive") c.Header("Access-Control-Allow-Origin", "*") } + var upstreamHeaders http.Header + committed := false + commitSSE := func() { + if committed { + return + } + setSSEHeaders() + handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + committed = true + } + writeKeepAlive := func() bool { + commitSSE() + _, _ = c.Writer.Write([]byte(": keep-alive\n\n")) + flusher.Flush() + return true + } + keepAliveC, stopKeepAlive := handlers.NewPreFirstTokenKeepAliveTicker(h.CurrentConfig()) + defer stopKeepAlive() + + dataChan, resolvedHeaders, errChan, okBootstrap := h.ExecuteStreamWithPreFTKeepAlive( + c, cliCtx, func(err error) { cliCancel(err) }, h.HandlerType(), modelName, rawJSON, h.GetAlt(c), keepAliveC, writeKeepAlive, + ) + if !okBootstrap { + return + } + upstreamHeaders = resolvedHeaders // Peek at the first chunk to determine success or failure before setting headers for { @@ -504,6 +530,14 @@ func (h *OpenAIAPIHandler) handleStreamingResponse(c *gin.Context, rawJSON []byt errChan = nil continue } + if committed { + stopKeepAlive() + pendingErrors := make(chan *interfaces.ErrorMessage, 1) + pendingErrors <- errMsg + close(pendingErrors) + h.handleStreamResult(c, flusher, func(err error) { cliCancel(err) }, make(chan []byte), pendingErrors, false) + return + } // Upstream failed immediately. Return proper error status and JSON. h.WriteErrorResponse(c, errMsg) if errMsg != nil { @@ -514,7 +548,15 @@ func (h *OpenAIAPIHandler) handleStreamingResponse(c *gin.Context, rawJSON []byt return case chunk, ok := <-dataChan: if !ok { + stopKeepAlive() if errMsg, hasPendingError := handlers.PendingStreamError(errChan); hasPendingError { + if committed { + pendingErrors := make(chan *interfaces.ErrorMessage, 1) + pendingErrors <- errMsg + close(pendingErrors) + h.handleStreamResult(c, flusher, func(err error) { cliCancel(err) }, make(chan []byte), pendingErrors, false) + return + } h.WriteErrorResponse(c, errMsg) if errMsg != nil { cliCancel(errMsg.Error) @@ -524,8 +566,7 @@ func (h *OpenAIAPIHandler) handleStreamingResponse(c *gin.Context, rawJSON []byt return } // Stream closed without data? Send DONE or just headers. - setSSEHeaders() - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + commitSSE() _, _ = fmt.Fprintf(c.Writer, "data: [DONE]\n\n") flusher.Flush() cliCancel(nil) @@ -533,15 +574,22 @@ func (h *OpenAIAPIHandler) handleStreamingResponse(c *gin.Context, rawJSON []byt } // Success! Commit to streaming headers. - setSSEHeaders() - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + stopKeepAlive() + commitSSE() + initialHasFinishReason := chunkHasFinishReason(chunk) _, _ = fmt.Fprintf(c.Writer, "data: %s\n\n", string(chunk)) flusher.Flush() + if termErr := chunkTerminalError(chunk); termErr != nil { + cliCancel(termErr.Error) + return + } // Continue streaming the rest - h.handleStreamResult(c, flusher, func(err error) { cliCancel(err) }, dataChan, errChan) + h.handleStreamResult(c, flusher, func(err error) { cliCancel(err) }, dataChan, errChan, initialHasFinishReason) return + case <-keepAliveC: + writeKeepAlive() } } } @@ -600,14 +648,38 @@ func (h *OpenAIAPIHandler) handleCompletionsStreamingResponse(c *gin.Context, ra modelName := gjson.GetBytes(chatCompletionsJSON, "model").String() cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) - dataChan, upstreamHeaders, errChan := h.ExecuteStreamWithAuthManager(cliCtx, h.HandlerType(), modelName, chatCompletionsJSON, "") - setSSEHeaders := func() { c.Header("Content-Type", "text/event-stream") c.Header("Cache-Control", "no-cache") c.Header("Connection", "keep-alive") c.Header("Access-Control-Allow-Origin", "*") } + var upstreamHeaders http.Header + committed := false + commitSSE := func() { + if committed { + return + } + setSSEHeaders() + handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + committed = true + } + writeKeepAlive := func() bool { + commitSSE() + _, _ = c.Writer.Write([]byte(": keep-alive\n\n")) + flusher.Flush() + return true + } + keepAliveC, stopKeepAlive := handlers.NewPreFirstTokenKeepAliveTicker(h.CurrentConfig()) + defer stopKeepAlive() + + dataChan, resolvedHeaders, errChan, okBootstrap := h.ExecuteStreamWithPreFTKeepAlive( + c, cliCtx, func(err error) { cliCancel(err) }, h.HandlerType(), modelName, chatCompletionsJSON, "", keepAliveC, writeKeepAlive, + ) + if !okBootstrap { + return + } + upstreamHeaders = resolvedHeaders // Peek at the first chunk for { @@ -621,6 +693,14 @@ func (h *OpenAIAPIHandler) handleCompletionsStreamingResponse(c *gin.Context, ra errChan = nil continue } + if committed { + stopKeepAlive() + pendingErrors := make(chan *interfaces.ErrorMessage, 1) + pendingErrors <- errMsg + close(pendingErrors) + h.handleStreamResult(c, flusher, func(err error) { cliCancel(err) }, make(chan []byte), pendingErrors, false) + return + } h.WriteErrorResponse(c, errMsg) if errMsg != nil { cliCancel(errMsg.Error) @@ -630,7 +710,15 @@ func (h *OpenAIAPIHandler) handleCompletionsStreamingResponse(c *gin.Context, ra return case chunk, ok := <-dataChan: if !ok { + stopKeepAlive() if errMsg, hasPendingError := handlers.PendingStreamError(errChan); hasPendingError { + if committed { + pendingErrors := make(chan *interfaces.ErrorMessage, 1) + pendingErrors <- errMsg + close(pendingErrors) + h.handleStreamResult(c, flusher, func(err error) { cliCancel(err) }, make(chan []byte), pendingErrors, false) + return + } h.WriteErrorResponse(c, errMsg) if errMsg != nil { cliCancel(errMsg.Error) @@ -639,8 +727,7 @@ func (h *OpenAIAPIHandler) handleCompletionsStreamingResponse(c *gin.Context, ra } return } - setSSEHeaders() - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + commitSSE() _, _ = fmt.Fprintf(c.Writer, "data: [DONE]\n\n") flusher.Flush() cliCancel(nil) @@ -648,12 +735,14 @@ func (h *OpenAIAPIHandler) handleCompletionsStreamingResponse(c *gin.Context, ra } // Success! Set headers. - setSSEHeaders() - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + stopKeepAlive() + commitSSE() // Write the first chunk converted := convertChatCompletionsStreamChunkToCompletions(chunk) + var initialHasFinishReason bool if converted != nil { + initialHasFinishReason = chunkHasFinishReason(converted) _, _ = fmt.Fprintf(c.Writer, "data: %s\n\n", string(converted)) flusher.Flush() } @@ -689,18 +778,65 @@ func (h *OpenAIAPIHandler) handleCompletionsStreamingResponse(c *gin.Context, ra h.handleStreamResult(c, flusher, func(err error) { stop() cliCancel(err) - }, convertedChan, errChan) + }, convertedChan, errChan, initialHasFinishReason) return + case <-keepAliveC: + writeKeepAlive() } } } -func (h *OpenAIAPIHandler) handleStreamResult(c *gin.Context, flusher http.Flusher, cancel func(error), data <-chan []byte, errs <-chan *interfaces.ErrorMessage) { +func chunkTerminalError(chunk []byte) *interfaces.ErrorMessage { + chunk = bytes.TrimSpace(chunk) + if bytes.HasPrefix(chunk, []byte("data:")) { + chunk = bytes.TrimSpace(chunk[5:]) + } + if gjson.GetBytes(chunk, "choices").Exists() || !gjson.GetBytes(chunk, "error").IsObject() { + return nil + } + msg := gjson.GetBytes(chunk, "error.message").String() + if msg == "" { + msg = string(chunk) + } + return &interfaces.ErrorMessage{ + StatusCode: http.StatusBadRequest, + Error: fmt.Errorf("%s", msg), + } +} + +func chunkHasFinishReason(chunk []byte) bool { + chunk = bytes.TrimSpace(chunk) + if bytes.HasPrefix(chunk, []byte("data:")) { + chunk = bytes.TrimSpace(chunk[5:]) + } + if chunkTerminalError(chunk) != nil { + return true + } + for _, choice := range gjson.GetBytes(chunk, "choices").Array() { + if reason := choice.Get("finish_reason"); reason.Exists() && reason.Type != gjson.Null && reason.String() != "" { + return true + } + } + return false +} + +func (h *OpenAIAPIHandler) handleStreamResult(c *gin.Context, flusher http.Flusher, cancel func(error), data <-chan []byte, errs <-chan *interfaces.ErrorMessage, initialHasFinishReason bool) { + sawFinishReason := initialHasFinishReason + var terminalChunkErr *interfaces.ErrorMessage h.ForwardStream(c, flusher, cancel, data, errs, handlers.StreamForwardOptions{ WriteChunk: func(chunk []byte) { + if termErr := chunkTerminalError(chunk); termErr != nil { + terminalChunkErr = termErr + sawFinishReason = true + } else if !sawFinishReason && chunkHasFinishReason(chunk) { + sawFinishReason = true + } _, _ = fmt.Fprintf(c.Writer, "data: %s\n\n", string(chunk)) }, + ChunkError: func() *interfaces.ErrorMessage { + return terminalChunkErr + }, WriteTerminalError: func(errMsg *interfaces.ErrorMessage) { - if errMsg == nil { + if errMsg == nil || terminalChunkErr != nil { return } status := http.StatusInternalServerError @@ -714,6 +850,15 @@ func (h *OpenAIAPIHandler) handleStreamResult(c *gin.Context, flusher http.Flush body := handlers.BuildErrorResponseBody(status, errText) _, _ = fmt.Fprintf(c.Writer, "data: %s\n\n", string(body)) }, + CloseError: func() *interfaces.ErrorMessage { + if sawFinishReason { + return nil + } + return &interfaces.ErrorMessage{ + StatusCode: http.StatusBadGateway, + Error: fmt.Errorf("upstream stream closed before any chunk carried finish_reason"), + } + }, WriteDone: func() { _, _ = fmt.Fprint(c.Writer, "data: [DONE]\n\n") }, diff --git a/sdk/api/handlers/openai/openai_images_handlers.go b/sdk/api/handlers/openai/openai_images_handlers.go index e91aae5bb..9c8e795d0 100644 --- a/sdk/api/handlers/openai/openai_images_handlers.go +++ b/sdk/api/handlers/openai/openai_images_handlers.go @@ -75,16 +75,20 @@ func setImagesSSEHeaders(c *gin.Context) { c.Header("Access-Control-Allow-Origin", "*") } -func (h *OpenAIAPIHandler) newImagesStreamKeepAliveTicker() (*time.Ticker, <-chan time.Time) { +func (h *OpenAIAPIHandler) newImagesStreamKeepAliveTicker() (func(), <-chan time.Time) { if h == nil || h.BaseAPIHandler == nil { - return nil, nil + return func() {}, nil } - interval := handlers.StreamingKeepAliveInterval(h.Cfg) - if interval <= 0 { - return nil, nil + keepAliveC, stop := handlers.NewStreamingKeepAliveTicker(handlers.StreamingKeepAliveInterval(h.CurrentConfig())) + return stop, keepAliveC +} + +func (h *OpenAIAPIHandler) newImagesPreFirstTokenKeepAliveTicker() (func(), <-chan time.Time) { + if h == nil || h.BaseAPIHandler == nil { + return func() {}, nil } - ticker := time.NewTicker(interval) - return ticker, ticker.C + keepAliveC, stop := handlers.NewPreFirstTokenKeepAliveTicker(h.CurrentConfig()) + return stop, keepAliveC } func writeImagesStreamKeepAlive(c *gin.Context, flusher http.Flusher) { @@ -109,23 +113,26 @@ func writeImagesStreamErrorEvent(c *gin.Context, errMsg *interfaces.ErrorMessage return errMsg } -func (h *OpenAIAPIHandler) waitImagesStreamExecution(c *gin.Context, flusher http.Flusher, execute func() imagesStreamExecutionResult) (imagesStreamExecutionResult, bool, bool) { +func (h *OpenAIAPIHandler) waitImagesStreamExecution(c *gin.Context, flusher http.Flusher, cliCancel handlers.APIHandlerCancelFunc, execute func() imagesStreamExecutionResult) (imagesStreamExecutionResult, bool, bool) { + stopKeepAlive, keepAliveC := h.newImagesPreFirstTokenKeepAliveTicker() + defer stopKeepAlive() + if keepAliveC == nil { + return execute(), false, false + } + resultChan := make(chan imagesStreamExecutionResult, 1) go func() { resultChan <- execute() }() - keepAlive, keepAliveC := h.newImagesStreamKeepAliveTicker() - defer func() { - if keepAlive != nil { - keepAlive.Stop() - } - }() - streamStarted := false for { select { case <-c.Request.Context().Done(): + if cliCancel != nil { + cliCancel(c.Request.Context().Err()) + } + <-resultChan return imagesStreamExecutionResult{}, streamStarted, true case result := <-resultChan: return result, streamStarted, false @@ -223,14 +230,46 @@ func isXAIImagesBaseModel(baseModel string) bool { } } +func registeredXAIImagesModel(model string) (string, bool) { + model = strings.TrimSpace(model) + if model == "" { + return "", false + } + modelRegistry := registry.GetGlobalRegistry() + for _, candidate := range []string{model, strings.ToLower(model)} { + for _, provider := range modelRegistry.GetModelProviders(candidate) { + if strings.EqualFold(strings.TrimSpace(provider), "xai") { + return candidate, true + } + } + } + return "", false +} + +func isXAIVendorAlias(prefix string) bool { + switch strings.ToLower(strings.TrimSpace(prefix)) { + case "xai", "x-ai", "grok": + return true + default: + return false + } +} + func isXAIImagesModel(model string) bool { prefix, baseModel := imagesModelParts(model) if !isXAIImagesBaseModel(baseModel) { return false } - prefix = strings.ToLower(strings.TrimSpace(prefix)) - return prefix == "" || prefix == "xai" || prefix == "x-ai" || prefix == "grok" + trimmedPrefix := strings.TrimSpace(prefix) + if trimmedPrefix == "" || isXAIVendorAlias(trimmedPrefix) { + return true + } + if _, ok := registeredXAIImagesModel(prefix + "/" + canonicalXAIImagesModel(model)); ok { + return true + } + _, ok := registeredXAIImagesModel(model) + return ok } func isSupportedImagesModel(model string) bool { @@ -291,6 +330,33 @@ func canonicalXAIImagesModel(model string) string { } } +func (h *OpenAIAPIHandler) routingXAIImagesModel(model string) string { + canonical := canonicalXAIImagesModel(model) + prefix, _ := imagesModelParts(model) + if prefix == "" { + return canonical + } + prefixedModel := prefix + "/" + canonical + if resolved, ok := registeredXAIImagesModel(prefixedModel); ok { + return resolved + } + if resolved, ok := registeredXAIImagesModel(model); ok { + return resolved + } + if h != nil && !isXAIVendorAlias(prefix) { + if cfg := h.CurrentConfig(); cfg != nil && cfg.ForceModelPrefix { + return prefixedModel + } + } + if h != nil && h.BaseAPIHandler != nil && h.AuthManager != nil && h.AuthManager.HomeEnabled() { + return prefixedModel + } + if _, ok := registeredXAIImagesModel(canonical); !ok { + return prefixedModel + } + return canonical +} + func xaiImagesAspectRatio(raw string, fallback string) string { switch strings.ToLower(strings.TrimSpace(raw)) { case "1:1", "square": @@ -617,14 +683,16 @@ func parseBoolField(raw string, fallback bool) bool { } func (h *OpenAIAPIHandler) ImagesGenerations(c *gin.Context) { - if h != nil && h.BaseAPIHandler != nil && h.BaseAPIHandler.Cfg != nil && h.BaseAPIHandler.Cfg.DisableImageGeneration == internalconfig.DisableImageGenerationAll { - c.AbortWithStatus(http.StatusNotFound) - return + if h != nil { + if cfg := h.CurrentConfig(); cfg != nil && cfg.DisableImageGeneration == internalconfig.DisableImageGenerationAll { + c.AbortWithStatus(http.StatusNotFound) + return + } } rawJSON, err := handlers.ReadRequestBody(c) if err != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ + c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), handlers.ErrorResponse{ Error: handlers.ErrorDetail{ Message: fmt.Sprintf("Invalid request: %v", err), Type: "invalid_request_error", @@ -674,7 +742,7 @@ func (h *OpenAIAPIHandler) ImagesGenerations(c *gin.Context) { } if isXAIImagesModel(imageModel) { xaiReq := buildXAIImagesGenerationsRequest(rawJSON, imageModel, responseFormat) - h.handleXAIImages(c, xaiReq, responseFormat, "image_generation", stream) + h.handleXAIImages(c, xaiReq, imageModel, responseFormat, "image_generation", stream) return } if isOpenAICompatImagesModel(imageModel) { @@ -721,9 +789,11 @@ func (h *OpenAIAPIHandler) ImagesGenerations(c *gin.Context) { } func (h *OpenAIAPIHandler) ImagesEdits(c *gin.Context) { - if h != nil && h.BaseAPIHandler != nil && h.BaseAPIHandler.Cfg != nil && h.BaseAPIHandler.Cfg.DisableImageGeneration == internalconfig.DisableImageGenerationAll { - c.AbortWithStatus(http.StatusNotFound) - return + if h != nil { + if cfg := h.CurrentConfig(); cfg != nil && cfg.DisableImageGeneration == internalconfig.DisableImageGenerationAll { + c.AbortWithStatus(http.StatusNotFound) + return + } } contentType := strings.ToLower(strings.TrimSpace(c.GetHeader("Content-Type"))) @@ -747,7 +817,7 @@ func (h *OpenAIAPIHandler) ImagesEdits(c *gin.Context) { func (h *OpenAIAPIHandler) imagesEditsFromMultipart(c *gin.Context) { form, err := c.MultipartForm() if err != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ + c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), handlers.ErrorResponse{ Error: handlers.ErrorDetail{ Message: fmt.Sprintf("Invalid request: %v", err), Type: "invalid_request_error", @@ -834,7 +904,7 @@ func (h *OpenAIAPIHandler) imagesEditsFromMultipart(c *gin.Context) { quality := strings.TrimSpace(c.PostForm("quality")) n := parseIntField(c.PostForm("n"), 0) xaiReq := buildXAIImagesEditRequest(imageModel, prompt, images, responseFormat, aspectRatio, resolution, quality, n) - h.handleXAIImages(c, xaiReq, responseFormat, "image_edit", stream) + h.handleXAIImages(c, xaiReq, imageModel, responseFormat, "image_edit", stream) return } if isOpenAICompatImagesModel(imageModel) { @@ -912,7 +982,7 @@ func (h *OpenAIAPIHandler) imagesEditsFromMultipart(c *gin.Context) { func (h *OpenAIAPIHandler) imagesEditsFromJSON(c *gin.Context) { rawJSON, err := handlers.ReadRequestBody(c) if err != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ + c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), handlers.ErrorResponse{ Error: handlers.ErrorDetail{ Message: fmt.Sprintf("Invalid request: %v", err), Type: "invalid_request_error", @@ -973,7 +1043,7 @@ func (h *OpenAIAPIHandler) imagesEditsFromJSON(c *gin.Context) { } aspectRatio, resolution, quality, n := xaiImagesEditOptionsFromJSON(rawJSON) xaiReq := buildXAIImagesEditRequest(imageModel, prompt, images, responseFormat, aspectRatio, resolution, quality, n) - h.handleXAIImages(c, xaiReq, responseFormat, "image_edit", stream) + h.handleXAIImages(c, xaiReq, imageModel, responseFormat, "image_edit", stream) return } if isOpenAICompatImagesModel(imageModel) { @@ -1160,12 +1230,12 @@ func buildImagesAPIResponseFromXAI(payload []byte, responseFormat string) ([]byt return out, nil } -func (h *OpenAIAPIHandler) handleXAIImages(c *gin.Context, xaiReq []byte, responseFormat string, streamPrefix string, stream bool) { +func (h *OpenAIAPIHandler) handleXAIImages(c *gin.Context, xaiReq []byte, imageModel string, responseFormat string, streamPrefix string, stream bool) { if stream { - h.streamXAIImages(c, xaiReq, responseFormat, streamPrefix) + h.streamXAIImages(c, xaiReq, imageModel, responseFormat, streamPrefix) return } - h.collectXAIImages(c, xaiReq, responseFormat) + h.collectXAIImages(c, xaiReq, imageModel, responseFormat) } func (h *OpenAIAPIHandler) handleOpenAICompatImages(c *gin.Context, compatReq []byte, imageModel string, responseFormat string, streamPrefix string, stream bool) { @@ -1224,22 +1294,21 @@ func (h *OpenAIAPIHandler) streamRoutedImages(c *gin.Context, imageReq []byte, i cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) cliCtx = handlers.WithDisallowFreeAuth(cliCtx) model := strings.TrimSpace(imageModel) - execution, streamStarted, canceled := h.waitImagesStreamExecution(c, flusher, func() imagesStreamExecutionResult { + execution, streamStarted, canceled := h.waitImagesStreamExecution(c, flusher, cliCancel, func() imagesStreamExecutionResult { dataChan, upstreamHeaders, errChan := h.ExecuteImageStreamWithAuthManager(cliCtx, xaiImagesHandlerType, model, imageReq, "") return imagesStreamExecutionResult{Data: dataChan, UpstreamHeaders: upstreamHeaders, Errs: errChan} }) if canceled { - cliCancel(c.Request.Context().Err()) return } dataChan := execution.Data upstreamHeaders := execution.UpstreamHeaders errChan := execution.Errs - keepAlive, keepAliveC := h.newImagesStreamKeepAliveTicker() + rawStopKeepAlive, keepAliveC := h.newImagesPreFirstTokenKeepAliveTicker() stopKeepAlive := func() { - if keepAlive != nil { - keepAlive.Stop() - keepAlive = nil + if rawStopKeepAlive != nil { + rawStopKeepAlive() + rawStopKeepAlive = nil keepAliveC = nil } } @@ -1306,12 +1375,8 @@ func (h *OpenAIAPIHandler) streamRoutedImages(c *gin.Context, imageReq []byte, i } func (h *OpenAIAPIHandler) forwardRawImageStream(ctx context.Context, c *gin.Context, cancel func(error), data <-chan []byte, errs <-chan *interfaces.ErrorMessage) { - keepAlive, keepAliveC := h.newImagesStreamKeepAliveTicker() - defer func() { - if keepAlive != nil { - keepAlive.Stop() - } - }() + stopKeepAlive, keepAliveC := h.newImagesStreamKeepAliveTicker() + defer stopKeepAlive() for { select { @@ -1370,22 +1435,21 @@ func (h *OpenAIAPIHandler) streamOpenAICompatImages(c *gin.Context, compatReq [] cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) model := strings.TrimSpace(imageModel) - execution, streamStarted, canceled := h.waitImagesStreamExecution(c, flusher, func() imagesStreamExecutionResult { + execution, streamStarted, canceled := h.waitImagesStreamExecution(c, flusher, cliCancel, func() imagesStreamExecutionResult { dataChan, upstreamHeaders, errChan := h.ExecuteImageStreamWithAuthManager(cliCtx, xaiImagesHandlerType, model, compatReq, "") return imagesStreamExecutionResult{Data: dataChan, UpstreamHeaders: upstreamHeaders, Errs: errChan} }) if canceled { - cliCancel(c.Request.Context().Err()) return } dataChan := execution.Data upstreamHeaders := execution.UpstreamHeaders errChan := execution.Errs - keepAlive, keepAliveC := h.newImagesStreamKeepAliveTicker() + rawStopKeepAlive, keepAliveC := h.newImagesPreFirstTokenKeepAliveTicker() stopKeepAlive := func() { - if keepAlive != nil { - keepAlive.Stop() - keepAlive = nil + if rawStopKeepAlive != nil { + rawStopKeepAlive() + rawStopKeepAlive = nil keepAliveC = nil } } @@ -1458,9 +1522,12 @@ func (h *OpenAIAPIHandler) streamOpenAICompatImages(c *gin.Context, compatReq [] } } -func (h *OpenAIAPIHandler) collectXAIImages(c *gin.Context, xaiReq []byte, responseFormat string) { - model := strings.TrimSpace(gjson.GetBytes(xaiReq, "model").String()) - h.collectImagesWithModel(c, xaiReq, model, responseFormat) +func (h *OpenAIAPIHandler) collectXAIImages(c *gin.Context, xaiReq []byte, imageModel string, responseFormat string) { + model := strings.TrimSpace(imageModel) + if model == "" { + model = strings.TrimSpace(gjson.GetBytes(xaiReq, "model").String()) + } + h.collectImagesWithModel(c, xaiReq, h.routingXAIImagesModel(model), responseFormat) } func (h *OpenAIAPIHandler) collectImagesWithModel(c *gin.Context, imageReq []byte, model string, responseFormat string) { @@ -1495,9 +1562,12 @@ func (h *OpenAIAPIHandler) collectImagesWithModel(c *gin.Context, imageReq []byt cliCancel(nil) } -func (h *OpenAIAPIHandler) streamXAIImages(c *gin.Context, xaiReq []byte, responseFormat string, streamPrefix string) { - model := strings.TrimSpace(gjson.GetBytes(xaiReq, "model").String()) - h.streamImagesWithModel(c, xaiReq, model, responseFormat, streamPrefix) +func (h *OpenAIAPIHandler) streamXAIImages(c *gin.Context, xaiReq []byte, imageModel string, responseFormat string, streamPrefix string) { + model := strings.TrimSpace(imageModel) + if model == "" { + model = strings.TrimSpace(gjson.GetBytes(xaiReq, "model").String()) + } + h.streamImagesWithModel(c, xaiReq, h.routingXAIImagesModel(model), responseFormat, streamPrefix) } func (h *OpenAIAPIHandler) streamImagesWithModel(c *gin.Context, imageReq []byte, model string, responseFormat string, streamPrefix string) { @@ -1525,11 +1595,11 @@ func (h *OpenAIAPIHandler) streamImagesWithModel(c *gin.Context, imageReq []byte resultChan <- imageStreamResult{resp: resp, upstreamHeaders: upstreamHeaders, errMsg: errMsg} }() - keepAlive, keepAliveC := h.newImagesStreamKeepAliveTicker() + rawStopKeepAlive, keepAliveC := h.newImagesPreFirstTokenKeepAliveTicker() stopKeepAlive := func() { - if keepAlive != nil { - keepAlive.Stop() - keepAlive = nil + if rawStopKeepAlive != nil { + rawStopKeepAlive() + rawStopKeepAlive = nil keepAliveC = nil } } @@ -1553,6 +1623,7 @@ func (h *OpenAIAPIHandler) streamImagesWithModel(c *gin.Context, imageReq []byte select { case <-c.Request.Context().Done(): cliCancel(c.Request.Context().Err()) + <-resultChan return case <-keepAliveC: setImagesSSEHeaders(c) @@ -1809,22 +1880,21 @@ func (h *OpenAIAPIHandler) streamImagesFromResponses(c *gin.Context, responsesRe if mainModel == "" { mainModel = defaultImagesMainModel } - execution, streamStarted, canceled := h.waitImagesStreamExecution(c, flusher, func() imagesStreamExecutionResult { + execution, streamStarted, canceled := h.waitImagesStreamExecution(c, flusher, cliCancel, func() imagesStreamExecutionResult { dataChan, upstreamHeaders, errChan := h.ExecuteStreamWithAuthManager(cliCtx, "openai-response", mainModel, responsesReq, "") return imagesStreamExecutionResult{Data: dataChan, UpstreamHeaders: upstreamHeaders, Errs: errChan} }) if canceled { - cliCancel(c.Request.Context().Err()) return } dataChan := execution.Data upstreamHeaders := execution.UpstreamHeaders errChan := execution.Errs - keepAlive, keepAliveC := h.newImagesStreamKeepAliveTicker() + rawStopKeepAlive, keepAliveC := h.newImagesPreFirstTokenKeepAliveTicker() stopKeepAlive := func() { - if keepAlive != nil { - keepAlive.Stop() - keepAlive = nil + if rawStopKeepAlive != nil { + rawStopKeepAlive() + rawStopKeepAlive = nil keepAliveC = nil } } @@ -1904,12 +1974,8 @@ func (h *OpenAIAPIHandler) forwardImagesStream(ctx context.Context, c *gin.Conte if responseFormat == "" { responseFormat = "b64_json" } - keepAlive, keepAliveC := h.newImagesStreamKeepAliveTicker() - defer func() { - if keepAlive != nil { - keepAlive.Stop() - } - }() + stopKeepAlive, keepAliveC := h.newImagesStreamKeepAliveTicker() + defer stopKeepAlive() emitError := func(errMsg *interfaces.ErrorMessage) *interfaces.ErrorMessage { errMsg = writeImagesStreamErrorEvent(c, errMsg) diff --git a/sdk/api/handlers/openai/openai_responses_handlers.go b/sdk/api/handlers/openai/openai_responses_handlers.go index 977aa6cf8..d0396925c 100644 --- a/sdk/api/handlers/openai/openai_responses_handlers.go +++ b/sdk/api/handlers/openai/openai_responses_handlers.go @@ -19,23 +19,29 @@ import ( "github.com/gin-gonic/gin" "github.com/router-for-me/CLIProxyAPI/v8/internal/client/codex/optimize-multi-agent-v2" + "github.com/router-for-me/CLIProxyAPI/v8/internal/clienterror" . "github.com/router-for-me/CLIProxyAPI/v8/internal/constant" + "github.com/router-for-me/CLIProxyAPI/v8/internal/httpwire" "github.com/router-for-me/CLIProxyAPI/v8/internal/interfaces" "github.com/router-for-me/CLIProxyAPI/v8/internal/registry" + sdkaccess "github.com/router-for-me/CLIProxyAPI/v8/sdk/access" "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers" + "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/usage" "github.com/tidwall/gjson" "github.com/tidwall/sjson" ) -func writeResponsesSSEChunk(w io.Writer, chunk []byte) { +func writeResponsesSSEChunk(w io.Writer, chunk []byte) error { if w == nil || len(chunk) == 0 { - return + return nil } - if _, err := w.Write(chunk); err != nil { - return + if n, errWrite := w.Write(chunk); errWrite != nil { + return errWrite + } else if n != len(chunk) { + return io.ErrShortWrite } if bytes.HasSuffix(chunk, []byte("\n\n")) || bytes.HasSuffix(chunk, []byte("\r\n\r\n")) { - return + return nil } suffix := []byte("\n\n") if bytes.HasSuffix(chunk, []byte("\r\n")) { @@ -43,9 +49,17 @@ func writeResponsesSSEChunk(w io.Writer, chunk []byte) { } else if bytes.HasSuffix(chunk, []byte("\n")) { suffix = []byte("\n") } - if _, err := w.Write(suffix); err != nil { - return + if n, errWrite := w.Write(suffix); errWrite != nil { + return errWrite + } else if n != len(suffix) { + return io.ErrShortWrite } + return nil +} + +// flushResponsesSSE preserves middleware behavior and transport flush errors. +func flushResponsesSSE(w http.ResponseWriter) error { + return httpwire.FlushResponse(w) } type responsesSSEFramer struct { @@ -59,6 +73,8 @@ type responsesSSEFramer struct { failureEvent string isCodexClient bool dataFrames int + writeErr error + canComplete func() bool } func (f *responsesSSEFramer) WriteChunk(w io.Writer, chunk []byte) { @@ -117,7 +133,9 @@ func (f *responsesSSEFramer) Flush(w io.Writer) { } func (f *responsesSSEFramer) writeFrame(w io.Writer, frame []byte) { - writeResponsesSSEChunk(w, f.repairFrame(frame)) + if f.writeErr == nil { + f.writeErr = writeResponsesSSEChunk(w, f.repairFrame(frame)) + } } func (f *responsesSSEFramer) shouldFilterPrivateEvent(streamEvent, payloadType string) bool { @@ -130,8 +148,8 @@ func (f *responsesSSEFramer) shouldFilterPrivateEvent(streamEvent, payloadType s return false } - // Always filter internal WebSocket timing telemetry from SSE streams. - if strings.HasPrefix(name, "responsesapi.") { + // Always filter internal WebSocket timing telemetry and Meta subscription usage from SSE streams. + if strings.HasPrefix(name, "responsesapi.") || name == "response.subscription_usage" { return true } @@ -228,8 +246,11 @@ func (f *responsesSSEFramer) repairErrorPayload(payload []byte) []byte { errMsg := responsesSSEPayloadErrorMessage(payload) status := errMsg.StatusCode f.terminalError = errMsg + isResponseFailedPayload := gjson.GetBytes(payload, "type").String() == "response.failed" failureEvent := f.failureEvent - if failureEvent != "response.failed" { + if isResponseFailedPayload || failureEvent == "response.failed" { + failureEvent = "response.failed" + } else { failureEvent = "error" } f.terminalEvent = failureEvent @@ -244,6 +265,19 @@ func (f *responsesSSEFramer) repairErrorPayload(payload []byte) []byte { } if failureEvent == "response.failed" { chunk := handlers.BuildOpenAIResponsesStreamFailedChunk(status, errText, seq) + if isResponseFailedPayload && gjson.GetBytes(payload, "response").IsObject() { + repaired := f.repairCompletedPayload(payload) + respRaw := []byte(gjson.GetBytes(repaired, "response").Raw) + respRaw, _ = sjson.SetBytes(respRaw, "status", "failed") + if sanitizedErr := gjson.GetBytes(chunk, "response.error"); sanitizedErr.Exists() { + respRaw, _ = sjson.SetRawBytes(respRaw, "error", []byte(sanitizedErr.Raw)) + } + env := []byte(`{"type":"response.failed","sequence_number":0,"response":{}}`) + env, _ = sjson.SetBytes(env, "sequence_number", seq) + if updated, errSet := sjson.SetRawBytes(env, "response", respRaw); errSet == nil { + chunk = updated + } + } return []byte(fmt.Sprintf("event: response.failed\ndata: %s\n\n", chunk)) } chunk := handlers.BuildOpenAIResponsesStreamErrorChunk(status, errText, seq) @@ -495,6 +529,7 @@ func responsesSSENeedsLineBreak(pending, chunk []byte) bool { // It holds a pool of clients to interact with the backend service. type OpenAIResponsesAPIHandler struct { *handlers.BaseAPIHandler + AccessManager *sdkaccess.Manager } // NewOpenAIResponsesAPIHandler creates a new OpenAIResponses API handlers instance. @@ -534,7 +569,11 @@ func (h *OpenAIResponsesAPIHandler) OpenAIResponsesModels(c *gin.Context) { } func (h *OpenAIResponsesAPIHandler) prepareCodexMultiAgentV2Tools(c *gin.Context, payload []byte) []byte { - if h == nil || h.Cfg == nil { + if h == nil { + return payload + } + cfg := h.CurrentConfig() + if cfg == nil { return payload } @@ -553,7 +592,7 @@ func (h *OpenAIResponsesAPIHandler) prepareCodexMultiAgentV2Tools(c *gin.Context requestCtx, requestHeaders, payload, - h.Cfg.Client.Codex.OptimizeMultiAgentV2, + cfg.Client.Codex.OptimizeMultiAgentV2, homeEnabled, ) if prepared && c != nil { @@ -563,7 +602,11 @@ func (h *OpenAIResponsesAPIHandler) prepareCodexMultiAgentV2Tools(c *gin.Context } func (h *OpenAIResponsesAPIHandler) prepareCodexOrphanDelegation(c *gin.Context, payload []byte) []byte { - if h == nil || h.Cfg == nil || !h.Cfg.CodexOrphanDelegationCompatibility { + if h == nil { + return payload + } + cfg := h.CurrentConfig() + if cfg == nil || !cfg.CodexOrphanDelegationCompatibility { return payload } requestCtx := context.Background() @@ -586,7 +629,7 @@ func (h *OpenAIResponsesAPIHandler) Responses(c *gin.Context) { rawJSON, err := handlers.ReadRequestBody(c) // If data retrieval fails, return a 400 Bad Request error. if err != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ + c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), handlers.ErrorResponse{ Error: handlers.ErrorDetail{ Message: fmt.Sprintf("Invalid request: %v", err), Type: "invalid_request_error", @@ -611,7 +654,7 @@ func (h *OpenAIResponsesAPIHandler) Responses(c *gin.Context) { func (h *OpenAIResponsesAPIHandler) Compact(c *gin.Context) { rawJSON, err := handlers.ReadRequestBody(c) if err != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ + c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), handlers.ErrorResponse{ Error: handlers.ErrorDetail{ Message: fmt.Sprintf("Invalid request: %v", err), Type: "invalid_request_error", @@ -702,21 +745,58 @@ func (h *OpenAIResponsesAPIHandler) handleStreamingResponse(c *gin.Context, rawJ // New core execution path modelName := gjson.GetBytes(rawJSON, "model").String() - cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) - dataChan, upstreamHeaders, errChan := h.ExecuteStreamWithAuthManager(cliCtx, h.HandlerType(), modelName, rawJSON, "") - + deliveryCtx, finishDelivery := usage.WithStreamDelivery(context.Background()) + defer finishDelivery(context.Canceled) + cliCtx, cancelExecution := h.GetContextWithCancel(h, c, deliveryCtx) + cliCancel := func(err error) { + finishDelivery(err) + cancelExecution(err) + } setSSEHeaders := func() { c.Header("Content-Type", "text/event-stream") c.Header("Cache-Control", "no-cache") c.Header("Connection", "keep-alive") c.Header("Access-Control-Allow-Origin", "*") } + var upstreamHeaders http.Header + committed := false + commitSSE := func() { + if committed { + return + } + setSSEHeaders() + handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + committed = true + } + writeKeepAlive := func() bool { + commitSSE() + if errWrite := writeResponsesSSEChunk(c.Writer, []byte(": keep-alive\n\n")); errWrite != nil { + cliCancel(errWrite) + return false + } + if errFlush := flushResponsesSSE(c.Writer); errFlush != nil { + cliCancel(errFlush) + return false + } + return true + } + keepAliveC, stopKeepAlive := handlers.NewPreFirstTokenKeepAliveTicker(h.CurrentConfig()) + defer stopKeepAlive() + + dataChan, resolvedHeaders, errChan, okBootstrap := h.ExecuteStreamWithPreFTKeepAlive( + c, cliCtx, cliCancel, h.HandlerType(), modelName, rawJSON, "", keepAliveC, writeKeepAlive, + ) + if !okBootstrap { + return + } + upstreamHeaders = resolvedHeaders + isCodexClient := isCodexResponsesClientRequest(c) failureEvent := "error" if isCodexClient { failureEvent = "response.failed" } - framer := &responsesSSEFramer{failureEvent: failureEvent, isCodexClient: isCodexClient} + framer := &responsesSSEFramer{failureEvent: failureEvent, isCodexClient: isCodexClient, canComplete: func() bool { return usage.StreamDeliverySupported(cliCtx) }} var initialOutput bytes.Buffer // Peek at the first complete SSE data frame. @@ -732,22 +812,32 @@ func (h *OpenAIResponsesAPIHandler) handleStreamingResponse(c *gin.Context, rawJ continue } framer.Flush(&initialOutput) - safeErrMsg := sanitizeResponsesStreamErrorMessage(errMsg) - if framer.dataFrames == 0 { - safeErrMsg = sanitizeResponsesInitialErrorMessage(errMsg) - } - if safeErrMsg != nil && framer.dataFrames > 0 { - setSSEHeaders() - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) - _, _ = c.Writer.Write(initialOutput.Bytes()) - flusher.Flush() + if committed || framer.dataFrames > 0 { + stopKeepAlive() + safeErrMsg := sanitizeResponsesStreamErrorMessage(errMsg) + commitSSE() + if initialOutput.Len() > 0 { + if errWrite := writeResponsesSSEChunk(c.Writer, initialOutput.Bytes()); errWrite != nil { + cliCancel(errWrite) + return + } + if errFlush := flushResponsesSSE(c.Writer); errFlush != nil { + cliCancel(errFlush) + return + } + } + if safeErrMsg == nil { + cliCancel(nil) + return + } pendingErrors := make(chan *interfaces.ErrorMessage, 1) pendingErrors <- safeErrMsg close(pendingErrors) h.forwardResponsesStream(c, flusher, func(err error) { cliCancel(err) }, make(chan []byte), pendingErrors, framer) return } - // Upstream failed before a complete SSE data frame. Return JSON. + // Upstream failed before SSE was committed. Return JSON with HTTP status. + safeErrMsg := sanitizeResponsesInitialErrorMessage(errMsg) h.LoggingAPIResponseError(context.WithValue(context.Background(), "gin", c), safeErrMsg) h.WriteErrorResponse(c, safeErrMsg) if safeErrMsg != nil { @@ -767,17 +857,25 @@ func (h *OpenAIResponsesAPIHandler) handleStreamingResponse(c *gin.Context, rawJ } errMsg = &interfaces.ErrorMessage{StatusCode: http.StatusBadGateway, Error: fmt.Errorf("%s", message)} } - if framer.dataFrames > 0 { + if committed || framer.dataFrames > 0 { errMsg = sanitizeResponsesStreamErrorMessage(errMsg) } else { errMsg = sanitizeResponsesInitialErrorMessage(errMsg) } - if framer.dataFrames > 0 { - setSSEHeaders() - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) - _, _ = c.Writer.Write(initialOutput.Bytes()) - flusher.Flush() + if committed || framer.dataFrames > 0 { + stopKeepAlive() + commitSSE() + if initialOutput.Len() > 0 { + if errWrite := writeResponsesSSEChunk(c.Writer, initialOutput.Bytes()); errWrite != nil { + cliCancel(errWrite) + return + } + if errFlush := flushResponsesSSE(c.Writer); errFlush != nil { + cliCancel(errFlush) + return + } + } if framer.terminalError != nil { h.logResponsesStreamError(c, framer, framer.terminalError) cliCancel(framer.terminalError.Error) @@ -809,10 +907,16 @@ func (h *OpenAIResponsesAPIHandler) handleStreamingResponse(c *gin.Context, rawJ continue } - setSSEHeaders() - handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) - _, _ = c.Writer.Write(initialOutput.Bytes()) - flusher.Flush() + stopKeepAlive() + commitSSE() + if errWrite := writeResponsesSSEChunk(c.Writer, initialOutput.Bytes()); errWrite != nil { + cliCancel(errWrite) + return + } + if errFlush := flushResponsesSSE(c.Writer); errFlush != nil { + cliCancel(errFlush) + return + } if framer.terminalError != nil { h.logResponsesStreamError(c, framer, framer.terminalError) cliCancel(framer.terminalError.Error) @@ -821,6 +925,10 @@ func (h *OpenAIResponsesAPIHandler) handleStreamingResponse(c *gin.Context, rawJ h.forwardResponsesStream(c, flusher, func(err error) { cliCancel(err) }, dataChan, errChan, framer) return + case <-keepAliveC: + if !writeKeepAlive() { + return + } } } } @@ -1017,6 +1125,18 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesStream(c *gin.Context, flush framer.failureEvent = "error" framer.isCodexClient = false } + // Initial frames have already been written and flushed by the caller. + if framer.writeErr != nil { + cancel(framer.writeErr) + return + } + if framer.canComplete != nil && framer.canComplete() && framer.terminalEvent == "response.completed" && framer.terminalError == nil { + if errMsg, pending := handlers.PendingStreamError(errs); pending { + h.logResponsesStreamError(c, framer, sanitizeResponsesStreamErrorMessage(errMsg)) + } + cancel(nil) + return + } writeTerminalError := func(errMsg *interfaces.ErrorMessage) { framer.Flush(c.Writer) if errMsg == nil { @@ -1049,10 +1169,17 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesStream(c *gin.Context, flush h.ForwardStream(c, flusher, cancel, data, errs, handlers.StreamForwardOptions{ NormalizeTerminalError: sanitizeResponsesStreamErrorMessage, + Flush: func() error { return flushResponsesSSE(c.Writer) }, WriteChunk: func(chunk []byte) { framer.WriteChunk(c.Writer, chunk) }, + ChunkDone: func() bool { + return framer.canComplete != nil && framer.canComplete() && framer.terminalEvent == "response.completed" && framer.writeErr == nil + }, ChunkError: func() *interfaces.ErrorMessage { + if framer.writeErr != nil { + return &interfaces.ErrorMessage{Error: framer.writeErr} + } if framer.terminalError != nil { h.logResponsesStreamError(c, framer, framer.terminalError) } @@ -1061,6 +1188,9 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesStream(c *gin.Context, flush WriteTerminalError: writeTerminalError, CloseError: func() *interfaces.ErrorMessage { framer.Flush(c.Writer) + if framer.writeErr != nil { + return &interfaces.ErrorMessage{Error: framer.writeErr} + } if framer.terminalError != nil { return framer.terminalError } diff --git a/sdk/api/handlers/openai/openai_responses_websocket.go b/sdk/api/handlers/openai/openai_responses_websocket.go index 280c3f55d..8807df989 100644 --- a/sdk/api/handlers/openai/openai_responses_websocket.go +++ b/sdk/api/handlers/openai/openai_responses_websocket.go @@ -1,9 +1,12 @@ package openai import ( + "bytes" "context" + "encoding/json" "errors" "fmt" + "io" "net" "net/http" "strings" @@ -20,6 +23,7 @@ import ( "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers" coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth" cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" + "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/usage" log "github.com/sirupsen/logrus" "github.com/tidwall/gjson" "github.com/tidwall/sjson" @@ -43,6 +47,17 @@ const ( codexLocalCompactionSummaryPrefix = "Another language model started to solve this problem and produced a summary of its thinking process. You also have access to the state of the tools that were used by that language model. Use this to build on the work that has already been done and avoid duplicating work. Here is the summary produced by the other language model, use the information in this summary to assist with your own analysis:" ) +var responsesWebsocketCompressionEnabled atomic.Bool + +func init() { + responsesWebsocketCompressionEnabled.Store(true) +} + +// SetResponsesWebsocketCompressionEnabled controls compression for future upgrades. +func SetResponsesWebsocketCompressionEnabled(enabled bool) { + responsesWebsocketCompressionEnabled.Store(enabled) +} + var responsesWebsocketUpgrader = websocket.Upgrader{ ReadBufferSize: 4096, WriteBufferSize: 4096, @@ -262,31 +277,111 @@ func truncateWebsocketCloseReason(reason string, maxBytes int) string { return truncated.String() } +func (h *OpenAIResponsesAPIHandler) authenticateResponsesWebsocketTurn(c *gin.Context, baseReq *http.Request, payload []byte, requestModelName string, updateRequestContext bool) (string, *interfaces.ErrorMessage) { + if h == nil || h.AccessManager == nil { + return "", nil + } + sourceReq := baseReq + if sourceReq == nil && c != nil { + sourceReq = c.Request + } + if sourceReq == nil { + return "", nil + } + authBody := payload + if requestModelName != "" && json.Valid(payload) && strings.TrimSpace(gjson.GetBytes(payload, "model").String()) == "" { + if updated, errSet := sjson.SetBytes(bytes.Clone(payload), "model", requestModelName); errSet == nil { + authBody = updated + } + } + req := sourceReq.Clone(sourceReq.Context()) + req.Method = http.MethodPost + if req.Header != nil { + for key := range req.Header { + lower := strings.ToLower(key) + if lower == "upgrade" || lower == "connection" || lower == "content-length" || strings.HasPrefix(lower, "sec-websocket-") { + delete(req.Header, key) + } + } + } + req.Body = io.NopCloser(bytes.NewReader(authBody)) + req.ContentLength = int64(len(authBody)) + req.GetBody = func() (io.ReadCloser, error) { + return io.NopCloser(bytes.NewReader(authBody)), nil + } + result, authErr := h.AccessManager.Authenticate(req.Context(), req) + if authErr != nil { + msg := strings.TrimSpace(authErr.Message) + if msg == "" { + msg = authErr.Error() + } + return "", &interfaces.ErrorMessage{ + StatusCode: authErr.HTTPStatusCode(), + Error: errors.New(msg), + } + } + if result == nil { + return "", nil + } + if c != nil && updateRequestContext { + c.Set("userApiKey", result.Principal) + c.Set("accessProvider", result.Provider) + if len(result.Metadata) > 0 { + c.Set("accessMetadata", result.Metadata) + } else { + c.Set("accessMetadata", nil) + } + if c.Request != nil { + reqCtx := usage.WithAPIKey(c.Request.Context(), result.Principal) + reqCtx = usage.WithAccessProvider(reqCtx, result.Provider) + reqCtx = usage.WithIsNativeKey(reqCtx, usage.IsNativeAccessProvider(result.Provider)) + c.Request = c.Request.WithContext(reqCtx) + } + } + return result.Principal, nil +} + // ResponsesWebsocket handles websocket requests for /v1/responses. // It accepts `response.create` and `response.append` requests and streams -// response events back as JSON websocket text messages. +// response events back as JSON websocket text messages. `response.interrupt` +// is forwarded to the current Codex upstream socket without starting a new turn. func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { - conn, err := responsesWebsocketUpgrader.Upgrade(c.Writer, c.Request, websocketUpgradeHeaders(c.Request)) + upgrader := responsesWebsocketUpgrader + upgrader.EnableCompression = responsesWebsocketCompressionEnabled.Load() + conn, err := upgrader.Upgrade(c.Writer, c.Request, websocketUpgradeHeaders(c.Request)) if err != nil { return } - var duplexInput <-chan cliproxyexecutor.WebsocketInput - if h != nil && h.Cfg != nil && h.Cfg.CodexResponseSteering { - socketCtx, cancelSocket := context.WithCancel(c.Request.Context()) - defer cancelSocket() - c.Request = c.Request.WithContext(socketCtx) - duplexInput = readResponsesWebsocketInput(socketCtx, cancelSocket, conn) - } writer := newResponsesWebsocketWriter(conn) passthroughSessionID := uuid.NewString() + socketCtx, cancelSocket := context.WithCancelCause(c.Request.Context()) + defer cancelSocket(nil) + c.Request = c.Request.WithContext(socketCtx) + upgradeRequest := c.Request.Clone(socketCtx) + // The reader owns the client socket so an in-flight interrupt is not stuck + // behind the response currently being forwarded. + requestLogEnabled := false + codexResponseSteering := false + if h != nil { + if cfg := h.CurrentConfig(); cfg != nil { + requestLogEnabled = cfg.RequestLog + codexResponseSteering = cfg.CodexResponseSteering + } + } + wsTimelineLog := newWebsocketTimelineLog(requestLogEnabled, websocketTimelineSourceFromContext(c)) + localInterrupt := newResponsesLocalInterrupt() + input, inputDone := readResponsesWebsocketInput(socketCtx, cancelSocket, conn, func(payload []byte) error { + return h.forwardResponsesWebsocketInterrupt(socketCtx, passthroughSessionID, payload) + }, localInterrupt, writer, wsTimelineLog) + var duplexInput <-chan cliproxyexecutor.WebsocketInput + if codexResponseSteering { + duplexInput = input + } downstreamSessionKey := websocketDownstreamSessionKey(c.Request) retainResponsesWebsocketToolCaches(downstreamSessionKey) clientIP := websocketClientAddress(c) log.Infof("responses websocket: client connected id=%s remote=%s", passthroughSessionID, clientIP) - requestLogEnabled := h != nil && h.Cfg != nil && h.Cfg.RequestLog - wsTimelineLog := newWebsocketTimelineLog(requestLogEnabled, websocketTimelineSourceFromContext(c)) - wsDone := make(chan struct{}) defer close(wsDone) var codexDuplexStream atomic.Bool @@ -335,10 +430,13 @@ func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { h.AuthManager.CloseExecutionSession(passthroughSessionID) log.Infof("responses websocket: upstream execution session closed id=%s", passthroughSessionID) } - wsTimelineLog.SetContext(c) + cancelSocket(nil) if errClose := conn.Close(); errClose != nil && !isWebsocketConnectionClosedError(errClose) { log.Warnf("responses websocket: close connection error: %v", errClose) } + // Finish control-frame diagnostics before snapshotting the timeline. + <-inputDone + wsTimelineLog.SetContext(c) }() var lastRequest []byte @@ -407,18 +505,18 @@ func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { var msgType int var payload []byte var errReadMessage error - if duplexInput == nil { - msgType, payload, errReadMessage = conn.ReadMessage() - } else { - select { - case message, ok := <-duplexInput: - if !ok { - return - } + select { + case message, ok := <-input: + if !ok { + errReadMessage = context.Cause(socketCtx) + } else { msgType, payload, errReadMessage = websocket.TextMessage, message.Payload, message.Err - case <-c.Request.Context().Done(): - return } + case <-socketCtx.Done(): + errReadMessage = context.Cause(socketCtx) + } + if errReadMessage != nil && errors.Is(errReadMessage, context.Canceled) { + return } if errReadMessage != nil { wsTerminateErr = errReadMessage @@ -450,6 +548,28 @@ func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { if requestModelName == "" { requestModelName = strings.TrimSpace(gjson.GetBytes(lastRequest, "model").String()) } + if _, errAuth := h.authenticateResponsesWebsocketTurn(c, upgradeRequest, payload, requestModelName, true); errAuth != nil { + h.LoggingAPIResponseError(context.WithValue(context.Background(), "gin", c), errAuth) + markAPIResponseTimestamp(c) + errorPayload, errWrite := writeResponsesWebsocketError(writer, wsTimelineLog, errAuth) + log.Infof( + "responses websocket: downstream_out id=%s type=%d event=%s payload=%s", + passthroughSessionID, + websocket.TextMessage, + websocketPayloadEventType(errorPayload), + websocketPayloadPreview(errorPayload), + ) + if errWrite != nil { + log.Warnf( + "responses websocket: downstream_out write failed id=%s event=%s error=%v", + passthroughSessionID, + websocketPayloadEventType(errorPayload), + errWrite, + ) + return + } + continue + } executionParent := context.WithValue(c.Request.Context(), "gin", c) executionParent, routeOverridesModelResolution := h.PrepareStreamModelRoute( executionParent, @@ -696,6 +816,20 @@ func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { current, ok := sessionAuthByID(authID) return ok && current != nil && !current.Disabled && current.Status != coreauth.StatusDisabled }) + boundPrincipal := usage.APIKeyFromContext(cliCtx) + cliCtx = cliproxyexecutor.WithWebsocketTurnAuthorizer(cliCtx, func(turnPayload []byte) ([]byte, error) { + principal, errAuth := h.authenticateResponsesWebsocketTurn(c, upgradeRequest, turnPayload, modelName, false) + if errAuth == nil && principal != "" && principal != boundPrincipal { + errAuth = &interfaces.ErrorMessage{ + StatusCode: http.StatusUnauthorized, + Error: errors.New("client credential changed during the websocket stream; reconnect"), + } + } + if errAuth == nil { + return nil, nil + } + return buildResponsesWebsocketErrorPayload(errAuth) + }) } if nativeWebsocketPassthrough && requestRequiresCurrentUpstreamWebsocket { cliCtx = cliproxyexecutor.WithRequiredUpstreamWebsocket(cliCtx) @@ -755,6 +889,7 @@ func (h *OpenAIResponsesAPIHandler) ResponsesWebsocket(c *gin.Context) { duplexStream: codexDuplexStream.Load, toolCacheTurn: toolCacheTurn, suppressError: replayPinnedAuthFailure, + localInterrupt: localInterrupt, }, ) if errForward != nil { diff --git a/sdk/api/handlers/openai/openai_responses_websocket_forward.go b/sdk/api/handlers/openai/openai_responses_websocket_forward.go index 4bcdfd685..8878a73ca 100644 --- a/sdk/api/handlers/openai/openai_responses_websocket_forward.go +++ b/sdk/api/handlers/openai/openai_responses_websocket_forward.go @@ -27,6 +27,7 @@ type responsesWebsocketForwardOptions struct { toolCacheTurn *responsesWebsocketToolCacheTurn suppressError func(*interfaces.ErrorMessage) bool keepAliveInterval *time.Duration + localInterrupt *responsesLocalInterrupt } func (h *OpenAIResponsesAPIHandler) forwardResponsesWebsocket( @@ -60,7 +61,7 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesWebsocket( var keepAliveC <-chan time.Time keepAliveInterval := time.Duration(0) if h != nil { - keepAliveInterval = handlers.StreamingKeepAliveInterval(h.Cfg) + keepAliveInterval = handlers.StreamingKeepAliveInterval(h.CurrentConfig()) } if opts.keepAliveInterval != nil { keepAliveInterval = *opts.keepAliveInterval @@ -70,9 +71,46 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesWebsocket( defer keepAliveTicker.Stop() keepAliveC = keepAliveTicker.C } + if opts.localInterrupt != nil { + opts.localInterrupt.begin() + defer opts.localInterrupt.end() + } + localFrames := opts.localInterrupt.framesChan() + + // handleErr ends the turn on a terminal stream error (non-nil errMsg). + handleErr := func(errMsg *interfaces.ErrorMessage) ([]byte, string, []string, *interfaces.ErrorMessage, error) { + h.LoggingAPIResponseError(context.WithValue(context.Background(), "gin", c), errMsg) + if opts.suppressError != nil && opts.suppressError(errMsg) { + cancel(errMsg.Error) + return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), errMsg, nil + } + markAPIResponseTimestamp(c) + if matched, errClose := writer.closeForUpstreamError(errMsg.Error); matched { + cancel(errMsg.Error) + if errClose != nil { + return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), errMsg, errClose + } + return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), errMsg, websocket.ErrCloseSent + } + + errorPayload, wrote, errTerminate := writeResponsesWebsocketTerminalError(writer, wsTimelineLog, errMsg, nil) + if wrote { + log.Infof( + "responses websocket: downstream_out id=%s type=%d event=%s payload=%s", + sessionID, + websocket.TextMessage, + websocketPayloadEventType(errorPayload), + websocketPayloadPreview(errorPayload), + ) + } + cancel(errMsg.Error) + return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), errMsg, errTerminate + } for { select { + case interruptPayload := <-localFrames: + return completeResponsesWebsocketLocalInterrupt(writer, wsTimelineLog, cancel, interruptPayload, outputItemsByIndex, outputItemsFallback, pendingToolCallIDs) case <-c.Request.Context().Done(): cancel(c.Request.Context().Err()) return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), nil, c.Request.Context().Err() @@ -90,36 +128,21 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesWebsocket( cancel(nil) return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), nil, nil } - - h.LoggingAPIResponseError(context.WithValue(context.Background(), "gin", c), errMsg) - if opts.suppressError != nil && opts.suppressError(errMsg) { - cancel(errMsg.Error) - return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), errMsg, nil - } - markAPIResponseTimestamp(c) - if matched, errClose := writer.closeForUpstreamError(errMsg.Error); matched { - cancel(errMsg.Error) - if errClose != nil { - return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), errMsg, errClose - } - return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), errMsg, websocket.ErrCloseSent - } - - errorPayload, wrote, errTerminate := writeResponsesWebsocketTerminalError(writer, wsTimelineLog, errMsg, nil) - if wrote { - log.Infof( - "responses websocket: downstream_out id=%s type=%d event=%s payload=%s", - sessionID, - websocket.TextMessage, - websocketPayloadEventType(errorPayload), - websocketPayloadPreview(errorPayload), - ) - } - cancel(errMsg.Error) - return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), errMsg, errTerminate + return handleErr(errMsg) case chunk, ok := <-data: if !ok { - if opts.duplexStream != nil && opts.duplexStream() { + duplex := opts.duplexStream != nil && opts.duplexStream() + if duplex || !completed { + // The stream producers send a terminal error before they close errs and then + // data, so it is already buffered here. The select picks between the closed + // data channel and that error at random; take the error. Otherwise a pinned-auth + // failure that needs a replay close (1012) ends as a plain disconnect (1006), + // and a duplex stream loses the error its executor reported. + if pendingErrMsg, hasPending := handlers.PendingStreamError(errs); hasPending { + return handleErr(pendingErrMsg) + } + } + if duplex { // A duplex stream ends with its socket, not an individual response. // The data channel may close before select observes its final error. _, errClose := writer.closeWithoutError() @@ -152,7 +175,11 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesWebsocket( payloads := websocketJSONPayloadsFromChunk(chunk) for i := range payloads { - if gjson.GetBytes(payloads[i], "type").String() == "response.created" { + eventType := gjson.GetBytes(payloads[i], "type").String() + if eventType == "response.subscription_usage" { + continue + } + if eventType == "response.created" { responseStarted = true completed = false outputItemsByIndex = make(map[int64][]byte) @@ -160,7 +187,6 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesWebsocket( pendingToolCallIDs = make(map[string]struct{}) } collectResponsesWebsocketOutputItem(payloads[i], outputItemsByIndex, &outputItemsFallback) - eventType := gjson.GetBytes(payloads[i], "type").String() if isResponsesWebsocketCompletionEvent(eventType) && (opts.preserveCompletionOutput == nil || !opts.preserveCompletionOutput()) { payloads[i] = restoreResponsesWebsocketCompletionOutput(payloads[i], outputItemsByIndex, outputItemsFallback) } @@ -184,7 +210,10 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesWebsocket( cancel(payloadErrMsg.Error) return completedOutput, completedResponseID, sortedStringSet(pendingToolCallIDs), payloadErrMsg, nil } - } else if isResponsesWebsocketCompletionEvent(eventType) { + } else if isResponsesWebsocketCompletionEvent(eventType) || eventType == "response.incomplete" { + // response.incomplete ends the turn too. Codex uses it, with + // reason "interrupted", to acknowledge response.interrupt. Closing + // the client socket here would block the follow-up request. completed = true completedOutput = responseCompletedOutputFromPayload(payloads[i], outputItemsByIndex, outputItemsFallback) completedResponseID = responseCompletedIDFromPayload(payloads[i]) @@ -233,6 +262,37 @@ func (h *OpenAIResponsesAPIHandler) forwardResponsesWebsocket( } } +func completeResponsesWebsocketLocalInterrupt( + writer *responsesWebsocketWriter, + wsTimelineLog websocketTimelineAppender, + cancel handlers.APIHandlerCancelFunc, + payload []byte, + outputItemsByIndex map[int64][]byte, + outputItemsFallback [][]byte, + pendingToolCallIDs map[string]struct{}, +) ([]byte, string, []string, *interfaces.ErrorMessage, error) { + responseID := strings.TrimSpace(gjson.GetBytes(payload, "response_id").String()) + output := responseCompletedOutputFromPayload([]byte(`{"response":{"output":[]}}`), outputItemsByIndex, outputItemsFallback) + incomplete := []byte(`{"type":"response.incomplete","response":{"status":"incomplete","incomplete_details":{"reason":"interrupted"}}}`) + var errSet error + incomplete, errSet = sjson.SetBytes(incomplete, "response.id", responseID) + if errSet != nil { + cancel(errSet) + return output, responseID, sortedStringSet(pendingToolCallIDs), nil, errSet + } + incomplete, errSet = sjson.SetRawBytes(incomplete, "response.output", output) + if errSet != nil { + cancel(errSet) + return output, responseID, sortedStringSet(pendingToolCallIDs), nil, errSet + } + if errWrite := writeResponsesWebsocketPayload(writer, wsTimelineLog, incomplete, time.Now()); errWrite != nil { + cancel(errWrite) + return output, responseID, sortedStringSet(pendingToolCallIDs), nil, errWrite + } + cancel(context.Canceled) + return output, responseID, sortedStringSet(pendingToolCallIDs), nil, nil +} + func responsesWebsocketErrorStatus(errMsg *interfaces.ErrorMessage) int { if errMsg == nil { return 0 diff --git a/sdk/api/handlers/openai/openai_responses_websocket_input.go b/sdk/api/handlers/openai/openai_responses_websocket_input.go index 0d378c043..cc232b3f8 100644 --- a/sdk/api/handlers/openai/openai_responses_websocket_input.go +++ b/sdk/api/handlers/openai/openai_responses_websocket_input.go @@ -1,27 +1,126 @@ package openai import ( + "bytes" "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "sync" + "time" "github.com/gorilla/websocket" + "github.com/router-for-me/CLIProxyAPI/v8/internal/interfaces" + coreauth "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/auth" cliproxyexecutor "github.com/router-for-me/CLIProxyAPI/v8/sdk/cliproxy/executor" + "github.com/tidwall/gjson" ) -// readResponsesWebsocketInput is the only downstream reader in duplex mode. +// responsesLocalInterrupt cancels an in-flight HTTP turn. A websocket upstream +// does not use it; that interrupt is written to the existing socket instead. +type responsesLocalInterrupt struct { + mu sync.Mutex + active bool + frames chan []byte +} + +func newResponsesLocalInterrupt() *responsesLocalInterrupt { + return &responsesLocalInterrupt{frames: make(chan []byte, 1)} +} + +func (s *responsesLocalInterrupt) begin() { + if s == nil { + return + } + s.mu.Lock() + s.active = true + s.mu.Unlock() +} + +func (s *responsesLocalInterrupt) end() { + if s == nil { + return + } + s.mu.Lock() + s.active = false + select { + case <-s.frames: + default: + } + s.mu.Unlock() +} + +func (s *responsesLocalInterrupt) deliver(payload []byte) bool { + if s == nil { + return false + } + s.mu.Lock() + defer s.mu.Unlock() + if !s.active { + return false + } + select { + case s.frames <- bytes.Clone(payload): + default: + } + return true +} + +func (s *responsesLocalInterrupt) framesChan() <-chan []byte { + if s == nil { + return nil + } + return s.frames +} + +// readResponsesWebsocketInput is the only downstream reader. Interrupts are +// handled immediately so they are not queued behind an active response. // The bounded queue backpressures clients instead of retaining unlimited input. -func readResponsesWebsocketInput(ctx context.Context, cancel context.CancelFunc, conn *websocket.Conn) <-chan cliproxyexecutor.WebsocketInput { +func readResponsesWebsocketInput(ctx context.Context, cancel context.CancelCauseFunc, conn *websocket.Conn, interrupt func([]byte) error, local *responsesLocalInterrupt, writer *responsesWebsocketWriter, timeline websocketTimelineAppender) (<-chan cliproxyexecutor.WebsocketInput, <-chan struct{}) { input := make(chan cliproxyexecutor.WebsocketInput, 16) + done := make(chan struct{}) go func() { + defer close(done) defer close(input) for { - kind, payload, err := conn.ReadMessage() - if err != nil { - cancel() + kind, payload, errRead := conn.ReadMessage() + if errRead != nil { + cancel(errRead) return } if kind != websocket.TextMessage && kind != websocket.BinaryMessage { continue } + // Control frames bypass response.create normalization. The original + // response_id, mode, and extension fields must reach upstream unchanged. + if json.Valid(payload) && gjson.GetBytes(payload, "type").String() == "response.interrupt" { + appendResponsesInterruptDiagnostic(timeline, payload, "") + errInterrupt := interrupt(payload) + switch { + case errInterrupt == nil: + appendResponsesInterruptDiagnostic(timeline, payload, "handled") + continue + case errors.Is(errInterrupt, cliproxyexecutor.ErrNoActiveUpstreamWebsocket) && local.deliver(payload): + appendResponsesInterruptDiagnostic(timeline, payload, "local_http") + continue + default: + appendResponsesInterruptDiagnostic(timeline, payload, "rejected") + // The sanitized outcome replaces raw error logging here: errors + // from transport dependencies may contain credentials or URLs. + _, errWrite := writeResponsesWebsocketError(writer, nil, &interfaces.ErrorMessage{ + StatusCode: http.StatusBadRequest, + Error: errInterrupt, + }) + if errWrite != nil { + cancel(errWrite) + return + } + continue + } + } select { case input <- cliproxyexecutor.WebsocketInput{Payload: payload}: case <-ctx.Done(): @@ -29,5 +128,57 @@ func readResponsesWebsocketInput(ctx context.Context, cancel context.CancelFunc, } } }() - return input + return input, done +} + +func appendResponsesInterruptDiagnostic(timeline websocketTimelineAppender, payload []byte, outcome string) { + if timeline == nil { + return + } + // Hash client-controlled IDs and omit all arbitrary fields, including mode. + // Only correlation metadata is needed to diagnose control-frame timing. + idHash := sha256.Sum256([]byte(gjson.GetBytes(payload, "response_id").String())) + diagnostic := struct { + Type string `json:"type"` + ResponseIDHash string `json:"response_id_hash"` + Outcome string `json:"outcome,omitempty"` + }{Type: "response.interrupt", ResponseIDHash: fmt.Sprintf("%x", idHash[:8]), Outcome: outcome} + data, errMarshal := json.Marshal(diagnostic) + if errMarshal != nil { + return + } + event := "interrupt" + if outcome != "" { + event = "interrupt.outcome" + } + timeline.Append(event, data, time.Now()) +} + +func (h *OpenAIResponsesAPIHandler) forwardResponsesWebsocketInterrupt(ctx context.Context, sessionID string, payload []byte) error { + responseID := gjson.GetBytes(payload, "response_id") + if responseID.Type != gjson.String || strings.TrimSpace(responseID.String()) == "" { + return fmt.Errorf("response.interrupt requires response_id") + } + if h == nil || h.AuthManager == nil { + return cliproxyexecutor.ErrNoActiveUpstreamWebsocket + } + exec, ok := h.AuthManager.Executor("codex") + if !ok || exec == nil { + return cliproxyexecutor.ErrNoActiveUpstreamWebsocket + } + sender, ok := exec.(interface { + InterruptExecutionSession(context.Context, string, []byte) error + }) + if !ok || sender == nil { + return fmt.Errorf("upstream executor does not support response.interrupt") + } + ctx = cliproxyexecutor.WithWebsocketAuthCheck(ctx, func(authID string) bool { + // Home runtime credentials live on the execution session, not in the global map. + auth, found := h.AuthManager.GetByID(authID) + if !found || auth == nil { + auth, found = h.AuthManager.GetExecutionSessionAuthByID(sessionID, authID) + } + return found && auth != nil && !auth.Disabled && auth.Status != coreauth.StatusDisabled + }) + return sender.InterruptExecutionSession(ctx, sessionID, payload) } diff --git a/sdk/api/handlers/openai/openai_responses_websocket_timeline.go b/sdk/api/handlers/openai/openai_responses_websocket_timeline.go index 77187a793..70ee202ee 100644 --- a/sdk/api/handlers/openai/openai_responses_websocket_timeline.go +++ b/sdk/api/handlers/openai/openai_responses_websocket_timeline.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "strings" + "sync" "time" "github.com/gin-gonic/gin" @@ -26,6 +27,7 @@ type responsesWebsocketPinnedAuthState struct { } type websocketTimelineLog struct { + mu sync.Mutex enabled bool source *requestlogging.FileBodySource builder *strings.Builder @@ -73,6 +75,12 @@ func (l *websocketTimelineLog) BeginRequest() { if l == nil || !l.enabled || l.source == nil { return } + l.mu.Lock() + defer l.mu.Unlock() + l.beginRequestLocked() +} + +func (l *websocketTimelineLog) beginRequestLocked() { l.closeCurrentPart() part, errCreate := l.source.CreatePart("request") if errCreate != nil { @@ -87,13 +95,15 @@ func (l *websocketTimelineLog) Append(eventType string, payload []byte, timestam if l == nil || !l.enabled { return } + l.mu.Lock() + defer l.mu.Unlock() data := formatWebsocketTimelineEvent(eventType, payload, timestamp) if len(data) == 0 { return } if l.source != nil { if l.currentPart == nil { - l.BeginRequest() + l.beginRequestLocked() } if l.currentPart == nil { return @@ -114,6 +124,8 @@ func (l *websocketTimelineLog) SetContext(c *gin.Context) { if l == nil || !l.enabled { return } + l.mu.Lock() + defer l.mu.Unlock() l.closeCurrentPart() if l.source != nil { if l.source.HasPayload() { @@ -133,6 +145,8 @@ func (l *websocketTimelineLog) String() string { if l == nil || !l.enabled { return "" } + l.mu.Lock() + defer l.mu.Unlock() l.closeCurrentPart() if l.source != nil { data, errRead := l.source.Bytes() diff --git a/sdk/api/handlers/openai/openai_speech_handlers.go b/sdk/api/handlers/openai/openai_speech_handlers.go new file mode 100644 index 000000000..5975f6f82 --- /dev/null +++ b/sdk/api/handlers/openai/openai_speech_handlers.go @@ -0,0 +1,250 @@ +package openai + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strings" + "unicode/utf8" + + "github.com/gin-gonic/gin" + "github.com/router-for-me/CLIProxyAPI/v8/internal/clienterror" + "github.com/router-for-me/CLIProxyAPI/v8/sdk/api/handlers" + "github.com/tidwall/gjson" +) + +const ( + xaiSpeechHandlerType = "openai-speech" + defaultXAISpeechModel = "grok-tts" + xaiSpeechVoiceModel = "grok-voice-tts-1.0" + defaultXAISpeechVoice = "eve" + maxXAISpeechRunes = 60000 + maxXAISpeechBody = 1 << 20 + defaultXAISpeechSampleRate = 24000 +) + +// openAISpeechVoices maps OpenAI TTS voice names onto Grok voices. +// Unknown names pass through so Grok voice ids keep working. +var openAISpeechVoices = map[string]string{ + "alloy": "ara", + "ash": "orion", + "ballad": "luna", + "coral": "celeste", + "echo": "rex", + "fable": "sal", + "onyx": "leo", + "nova": "eve", + "sage": "iris", + "shimmer": "aurora", + "verse": "lumen", +} + +func (h *OpenAIAPIHandler) AudioSpeech(c *gin.Context) { + h.handleXAISpeech(c) +} + +func (h *OpenAIAPIHandler) XAITTS(c *gin.Context) { + h.handleXAISpeech(c) +} + +func (h *OpenAIAPIHandler) handleXAISpeech(c *gin.Context) { + raw, err := handlers.ReadRequestBody(c) + if err != nil { + writeSpeechError(c, clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), fmt.Sprintf("Invalid request: %v", err)) + return + } + if len(raw) > maxXAISpeechBody { + writeSpeechError(c, http.StatusBadRequest, "request body is larger than 1MB") + return + } + + requested := strings.TrimSpace(gjson.GetBytes(raw, "model").String()) + model, ok := speechRoutingModel(requested) + if !ok { + writeSpeechError(c, http.StatusBadRequest, fmt.Sprintf("Model %s is not supported on /v1/audio/speech. Use %s.", requested, defaultXAISpeechModel)) + return + } + payload, format, err := buildXAISpeechPayload(raw) + if err != nil { + writeSpeechError(c, http.StatusBadRequest, err.Error()) + return + } + + cliCtx, cliCancel := h.GetContextWithCancel(h, c, context.Background()) + // Keep-alive newlines would be prefixed onto the audio body. + resp, upstreamHeaders, errMsg := h.ExecuteWithAuthManager(cliCtx, xaiSpeechHandlerType, model, payload, "") + if errMsg != nil { + h.WriteErrorResponse(c, errMsg) + if errMsg.Error != nil { + cliCancel(errMsg.Error) + } else { + cliCancel(nil) + } + return + } + + c.Header("Content-Type", speechResponseContentType(format, upstreamHeaders.Get("Content-Type"))) + handlers.WriteUpstreamHeaders(c.Writer.Header(), upstreamHeaders) + c.Writer.Header().Del("Content-Length") + _, _ = c.Writer.Write(resp) + cliCancel(nil) +} + +func writeSpeechError(c *gin.Context, status int, message string) { + c.JSON(status, handlers.ErrorResponse{ + Error: handlers.ErrorDetail{ + Message: message, + Type: "invalid_request_error", + }, + }) +} + +func speechRoutingModel(model string) (string, bool) { + switch speechModelBase(model) { + case "", "tts-1", "tts-1-hd", "gpt-4o-mini-tts", defaultXAISpeechModel: + return defaultXAISpeechModel, true + case xaiSpeechVoiceModel: + return xaiSpeechVoiceModel, true + default: + return "", false + } +} + +func speechModelBase(model string) string { + model = strings.TrimSpace(model) + lower := strings.ToLower(model) + for _, prefix := range []string{"xai/", "x-ai/", "grok/"} { + if strings.HasPrefix(lower, prefix) { + model = strings.TrimSpace(model[len(prefix):]) + break + } + } + if open := strings.LastIndex(model, "("); open > 0 && strings.HasSuffix(model, ")") { + model = strings.TrimSpace(model[:open]) + } + return strings.ToLower(strings.TrimSpace(model)) +} + +func buildXAISpeechPayload(raw []byte) ([]byte, string, error) { + if !gjson.ValidBytes(raw) { + return nil, "", errors.New("body must be valid JSON") + } + text := strings.TrimSpace(gjson.GetBytes(raw, "input").String()) + if text == "" { + text = strings.TrimSpace(gjson.GetBytes(raw, "text").String()) + } + if text == "" { + return nil, "", errors.New("input is required") + } + if utf8.RuneCountInString(text) > maxXAISpeechRunes { + return nil, "", errors.New("input is longer than 60000 characters") + } + + voice := strings.TrimSpace(gjson.GetBytes(raw, "voice").String()) + if voice == "" { + voice = gjson.GetBytes(raw, "voice_id").String() + } + language := strings.TrimSpace(gjson.GetBytes(raw, "language").String()) + if language == "" { + language = "auto" + } + body := map[string]any{ + "text": text, + "voice_id": mapXAISpeechVoice(voice), + "language": language, + } + if speed := gjson.GetBytes(raw, "speed"); speed.Exists() && speed.Type == gjson.Number && speed.Float() > 0 { + body["speed"] = speed.Float() + } + format, output, err := speechOutputFormat(raw) + if err != nil { + return nil, "", err + } + if output != nil { + body["output_format"] = output + } + encoded, err := json.Marshal(body) + if err != nil { + return nil, "", err + } + return encoded, format, nil +} + +func mapXAISpeechVoice(voice string) string { + voice = strings.ToLower(strings.TrimSpace(voice)) + if voice == "" { + return defaultXAISpeechVoice + } + if mapped, ok := openAISpeechVoices[voice]; ok { + return mapped + } + return voice +} + +func speechOutputFormat(raw []byte) (string, map[string]any, error) { + responseFormat := strings.ToLower(strings.TrimSpace(gjson.GetBytes(raw, "response_format").String())) + if responseFormat != "" { + format, output, err := speechCodecFormat(responseFormat, defaultXAISpeechSampleRate) + return format, output, err + } + native := gjson.GetBytes(raw, "output_format") + if !native.Exists() { + return "mp3", nil, nil + } + if !native.IsObject() { + return "", nil, errors.New("output_format must be an object") + } + codec := strings.ToLower(strings.TrimSpace(native.Get("codec").String())) + sampleRate := defaultXAISpeechSampleRate + if rate := native.Get("sample_rate"); rate.Exists() && rate.Type == gjson.Number && rate.Float() > 0 { + sampleRate = int(rate.Float()) + } + return speechCodecFormat(codec, sampleRate) +} + +func speechCodecFormat(codec string, sampleRate int) (string, map[string]any, error) { + switch codec { + case "", "mp3": + return "mp3", nil, nil + case "wav", "pcm": + if sampleRate <= 0 { + sampleRate = defaultXAISpeechSampleRate + } + return codec, map[string]any{"codec": codec, "sample_rate": sampleRate}, nil + default: + return "", nil, errors.New("response_format must be mp3, wav, or pcm") + } +} + +func speechContentType(format string) string { + switch format { + case "wav": + return "audio/wav" + case "pcm": + return "audio/pcm" + default: + return "audio/mpeg" + } +} + +func speechContentTypeNeedsDefault(contentType string) bool { + mediaType := strings.ToLower(strings.TrimSpace(contentType)) + if i := strings.Index(mediaType, ";"); i >= 0 { + mediaType = strings.TrimSpace(mediaType[:i]) + } + switch mediaType { + case "", "application/json", "application/octet-stream", "text/plain": + return true + default: + return false + } +} + +func speechResponseContentType(format, upstream string) string { + if !speechContentTypeNeedsDefault(upstream) { + return upstream + } + return speechContentType(format) +} diff --git a/sdk/api/handlers/openai/openai_videos_handlers.go b/sdk/api/handlers/openai/openai_videos_handlers.go index 1c556b7e3..55cd0a3d5 100644 --- a/sdk/api/handlers/openai/openai_videos_handlers.go +++ b/sdk/api/handlers/openai/openai_videos_handlers.go @@ -255,6 +255,15 @@ func readXAIVideosNativeRequest(c *gin.Context) ([]byte, error) { } func videosCreateRequestFromForm(c *gin.Context) ([]byte, error) { + if strings.EqualFold(strings.TrimSpace(c.ContentType()), "multipart/form-data") { + if _, errForm := c.MultipartForm(); errForm != nil { + return nil, errForm + } + } else if c != nil && c.Request != nil { + if errForm := c.Request.ParseForm(); errForm != nil { + return nil, errForm + } + } rawJSON := []byte(`{}`) for _, field := range []string{"model", "prompt", "seconds", "size", "aspect_ratio", "resolution"} { if value := strings.TrimSpace(c.PostForm(field)); value != "" { @@ -287,11 +296,13 @@ func firstPostForm(c *gin.Context, keys ...string) string { } func (h *OpenAIAPIHandler) videoAuthBindingTTL() time.Duration { - if h != nil && h.BaseAPIHandler != nil && h.Cfg != nil { - raw := strings.TrimSpace(h.Cfg.VideoResultAuthCacheTTL) - if raw != "" { - if ttl, err := time.ParseDuration(raw); err == nil && ttl > 0 { - return ttl + if h != nil { + if cfg := h.CurrentConfig(); cfg != nil { + raw := strings.TrimSpace(cfg.VideoResultAuthCacheTTL) + if raw != "" { + if ttl, err := time.ParseDuration(raw); err == nil && ttl > 0 { + return ttl + } } } } @@ -692,7 +703,7 @@ func openAIVideoStatus(status string) string { func (h *OpenAIAPIHandler) VideosCreate(c *gin.Context) { rawJSON, err := readVideosCreateRequest(c) if err != nil { - writeVideosFailedError(c, http.StatusBadRequest, defaultXAIVideosModel, "invalid_request_error", fmt.Sprintf("Invalid request: %v", err)) + writeVideosFailedError(c, clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), defaultXAIVideosModel, "invalid_request_error", fmt.Sprintf("Invalid request: %v", err)) return } @@ -728,7 +739,7 @@ func (h *OpenAIAPIHandler) XAIVideosExtensions(c *gin.Context) { func (h *OpenAIAPIHandler) handleXAIVideosNativePost(c *gin.Context) { rawJSON, err := readXAIVideosNativeRequest(c) if err != nil { - c.JSON(http.StatusBadRequest, handlers.ErrorResponse{ + c.JSON(clienterror.HTTPStatusFromErrorOr(err, http.StatusBadRequest), handlers.ErrorResponse{ Error: handlers.ErrorDetail{ Message: fmt.Sprintf("Invalid request: %v", err), Type: "invalid_request_error", @@ -938,8 +949,10 @@ func (h *OpenAIAPIHandler) videoContentHTTPClient(c *gin.Context) *http.Client { ctx = c.Request.Context() } var cfg *config.Config - if h != nil && h.BaseAPIHandler != nil && h.Cfg != nil { - cfg = &config.Config{SDKConfig: *h.Cfg} + if h != nil { + if sdkCfg := h.CurrentConfig(); sdkCfg != nil { + cfg = &config.Config{SDKConfig: *sdkCfg} + } } return helps.NewProxyAwareHTTPClient(ctx, cfg, h.videoContentDownloadAuth(c), 0) } diff --git a/sdk/api/handlers/openai_responses_stream_error.go b/sdk/api/handlers/openai_responses_stream_error.go index 1b8fceb9d..00d98e873 100644 --- a/sdk/api/handlers/openai_responses_stream_error.go +++ b/sdk/api/handlers/openai_responses_stream_error.go @@ -153,17 +153,28 @@ func BuildOpenAIResponsesStreamErrorChunk(status int, errText string, sequenceNu return []byte(`{"type":"error","error":{"type":"server_error","code":"internal_server_error","message":"internal error","param":null},"sequence_number":0}`) } +func applyOpenAIResponsesNestedErrorCode(errorDetail map[string]any) map[string]any { + if existing, ok := errorDetail["code"]; ok && existing != nil && strings.TrimSpace(fmt.Sprint(existing)) != "" { + return errorDetail + } + // Anthropic reports input overflow by message only; Responses clients key on this code. + if m, _ := errorDetail["message"].(string); errorDetail["type"] == "invalid_request_error" && strings.HasPrefix(m, "prompt is too long") { + errorDetail["code"] = "context_length_exceeded" + } + return errorDetail +} + func openAIResponsesStreamErrorDetail(status int, errText, code, message string) map[string]any { var payload map[string]any trimmed := strings.TrimSpace(errText) if trimmed != "" && json.Valid([]byte(trimmed)) { if errUnmarshal := unmarshalJSONWithNumber([]byte(trimmed), &payload); errUnmarshal == nil { if errorDetail, ok := payload["error"].(map[string]any); ok { - return errorDetail + return applyOpenAIResponsesNestedErrorCode(errorDetail) } if response, ok := payload["response"].(map[string]any); ok { if errorDetail, ok := response["error"].(map[string]any); ok { - return errorDetail + return applyOpenAIResponsesNestedErrorCode(errorDetail) } } if m, ok := payload["message"].(string); ok && strings.TrimSpace(m) != "" { diff --git a/sdk/api/handlers/request_body.go b/sdk/api/handlers/request_body.go index 568872d2b..3a16f38d5 100644 --- a/sdk/api/handlers/request_body.go +++ b/sdk/api/handlers/request_body.go @@ -3,17 +3,27 @@ package handlers import ( "bytes" "encoding/json" + "errors" "fmt" "io" + "math" + "net/http" "strings" "github.com/gin-gonic/gin" "github.com/klauspost/compress/zstd" ) +// MaxRequestBodyBytesContextKey stores the configured request body size limit (in bytes) on the Gin context. +const MaxRequestBodyBytesContextKey = "maxRequestBodyBytes" + // ReadRequestBody reads the incoming request body and decodes supported // Content-Encoding values before handlers inspect JSON fields. func ReadRequestBody(c *gin.Context) ([]byte, error) { + maxBytes := int64(0) + if c != nil { + maxBytes = c.GetInt64(MaxRequestBodyBytesContextKey) + } raw, err := c.GetRawData() if err != nil { return nil, err @@ -27,8 +37,12 @@ func ReadRequestBody(c *gin.Context) ([]byte, error) { return raw, nil } - decoded, err := decodeRequestBody(raw, encoding) + decoded, err := decodeRequestBody(raw, encoding, maxBytes) if err != nil { + var maxBytesErr *http.MaxBytesError + if errors.As(err, &maxBytesErr) { + return nil, err + } if json.Valid(raw) { return raw, nil } @@ -37,7 +51,7 @@ func ReadRequestBody(c *gin.Context) ([]byte, error) { return decoded, nil } -func decodeRequestBody(raw []byte, encoding string) ([]byte, error) { +func decodeRequestBody(raw []byte, encoding string, maxBytes int64) ([]byte, error) { parts := strings.Split(encoding, ",") body := raw for i := len(parts) - 1; i >= 0; i-- { @@ -46,7 +60,7 @@ func decodeRequestBody(raw []byte, encoding string) ([]byte, error) { case "", "identity": continue case "zstd": - decoded, err := decodeZstdRequestBody(body) + decoded, err := decodeZstdRequestBody(body, maxBytes) if err != nil { return nil, err } @@ -58,16 +72,23 @@ func decodeRequestBody(raw []byte, encoding string) ([]byte, error) { return body, nil } -func decodeZstdRequestBody(raw []byte) ([]byte, error) { +func decodeZstdRequestBody(raw []byte, maxBytes int64) ([]byte, error) { decoder, err := zstd.NewReader(bytes.NewReader(raw)) if err != nil { return nil, fmt.Errorf("failed to create zstd request decoder: %w", err) } defer decoder.Close() - decoded, err := io.ReadAll(decoder) + var reader io.Reader = decoder + if maxBytes > 0 && maxBytes < math.MaxInt64 { + reader = io.LimitReader(decoder, maxBytes+1) + } + decoded, err := io.ReadAll(reader) if err != nil { return nil, fmt.Errorf("failed to decode zstd request body: %w", err) } + if maxBytes > 0 && int64(len(decoded)) > maxBytes { + return nil, &http.MaxBytesError{Limit: maxBytes} + } return decoded, nil } diff --git a/sdk/api/handlers/stream_forwarder.go b/sdk/api/handlers/stream_forwarder.go index 7a8d2aaac..fb5828fc1 100644 --- a/sdk/api/handlers/stream_forwarder.go +++ b/sdk/api/handlers/stream_forwarder.go @@ -6,8 +6,62 @@ import ( "github.com/gin-gonic/gin" "github.com/router-for-me/CLIProxyAPI/v8/internal/interfaces" + "golang.org/x/net/context" ) +// ExecuteStreamWithPreFTKeepAlive executes a streaming request via ExecuteStreamWithAuthManager. +// When keepAliveC is non-nil (streaming.send-keepalive-before-ft is enabled and keepalive-seconds > 0), +// it runs the synchronous bootstrap in a goroutine and invokes writeKeepAlive on each tick while waiting. +// It returns (dataChan, upstreamHeaders, errChan, ok), where ok is false if the request context was canceled +// or writeKeepAlive returned false. +func (h *BaseAPIHandler) ExecuteStreamWithPreFTKeepAlive( + c *gin.Context, + cliCtx context.Context, + cliCancel func(error), + handlerType, modelName string, + rawJSON []byte, + alt string, + keepAliveC <-chan time.Time, + writeKeepAlive func() bool, +) (<-chan []byte, http.Header, <-chan *interfaces.ErrorMessage, bool) { + if keepAliveC == nil { + dataChan, upstreamHeaders, errChan := h.ExecuteStreamWithAuthManager(cliCtx, handlerType, modelName, rawJSON, alt) + return dataChan, upstreamHeaders, errChan, true + } + + type streamBootstrapResult struct { + data <-chan []byte + headers http.Header + errs <-chan *interfaces.ErrorMessage + } + bootstrapResult := make(chan streamBootstrapResult, 1) + go func() { + data, headers, errs := h.ExecuteStreamWithAuthManager(cliCtx, handlerType, modelName, rawJSON, alt) + bootstrapResult <- streamBootstrapResult{data: data, headers: headers, errs: errs} + }() + + for { + select { + case <-c.Request.Context().Done(): + if cliCancel != nil { + cliCancel(c.Request.Context().Err()) + } + <-bootstrapResult + return nil, nil, nil, false + case result := <-bootstrapResult: + return result.data, result.headers, result.errs, true + case <-keepAliveC: + if writeKeepAlive != nil && !writeKeepAlive() { + if cliCancel != nil { + cliCancel(context.Canceled) + } + <-bootstrapResult + return nil, nil, nil, false + } + } + } +} + // PendingStreamError returns an immediately available non-nil stream error. func PendingStreamError(errs <-chan *interfaces.ErrorMessage) (*interfaces.ErrorMessage, bool) { if errs == nil { @@ -35,6 +89,12 @@ type StreamForwardOptions struct { // The failure is passed to cancel without writing another terminal payload. ChunkError func() *interfaces.ErrorMessage + // ChunkDone reports successful terminal delivery after WriteChunk and Flush. + ChunkDone func() bool + + // Flush optionally exposes transport flush errors hidden by http.Flusher. + Flush func() error + // NormalizeTerminalError optionally replaces an upstream error before it is // written or passed to cancel. NormalizeTerminalError func(errMsg *interfaces.ErrorMessage) *interfaces.ErrorMessage @@ -64,6 +124,18 @@ func (h *BaseAPIHandler) ForwardStream(c *gin.Context, flusher http.Flusher, can return } + flush := func() bool { + if opts.Flush != nil { + if errFlush := opts.Flush(); errFlush != nil { + cancel(errFlush) + return false + } + } else { + flusher.Flush() + } + return true + } + writeChunk := opts.WriteChunk if writeChunk == nil { writeChunk = func([]byte) {} @@ -76,17 +148,12 @@ func (h *BaseAPIHandler) ForwardStream(c *gin.Context, flusher http.Flusher, can } } - keepAliveInterval := StreamingKeepAliveInterval(h.Cfg) + keepAliveInterval := StreamingKeepAliveInterval(h.CurrentConfig()) if opts.KeepAliveInterval != nil { keepAliveInterval = *opts.KeepAliveInterval } - var keepAlive *time.Ticker - var keepAliveC <-chan time.Time - if keepAliveInterval > 0 { - keepAlive = time.NewTicker(keepAliveInterval) - defer keepAlive.Stop() - keepAliveC = keepAlive.C - } + keepAliveC, stopKeepAlive := NewStreamingKeepAliveTicker(keepAliveInterval) + defer stopKeepAlive() var terminalErr *interfaces.ErrorMessage for { @@ -112,19 +179,25 @@ func (h *BaseAPIHandler) ForwardStream(c *gin.Context, flusher http.Flusher, can if opts.WriteTerminalError != nil { opts.WriteTerminalError(terminalErr) } - flusher.Flush() + if !flush() { + return + } cancel(terminalErr.Error) return } if opts.WriteDone != nil { opts.WriteDone() } - flusher.Flush() + if !flush() { + return + } cancel(nil) return } writeChunk(chunk) - flusher.Flush() + if !flush() { + return + } if opts.ChunkError != nil { chunkErr := opts.ChunkError() if chunkErr != nil { @@ -139,6 +212,10 @@ func (h *BaseAPIHandler) ForwardStream(c *gin.Context, flusher http.Flusher, can return } } + if opts.ChunkDone != nil && opts.ChunkDone() { + cancel(nil) + return + } case errMsg, ok := <-errs: if !ok { errs = nil @@ -151,7 +228,9 @@ func (h *BaseAPIHandler) ForwardStream(c *gin.Context, flusher http.Flusher, can } if opts.WriteTerminalError != nil { opts.WriteTerminalError(terminalErr) - flusher.Flush() + if !flush() { + return + } } } var execErr error @@ -162,7 +241,9 @@ func (h *BaseAPIHandler) ForwardStream(c *gin.Context, flusher http.Flusher, can return case <-keepAliveC: writeKeepAlive() - flusher.Flush() + if !flush() { + return + } } } }