package openai import ( "bytes" "io" "net/http" "net/http/httptest" "slices" "strings" "testing" "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/constant" relaycommon "github.com/QuantumNous/new-api/relay/common" relayconstant "github.com/QuantumNous/new-api/relay/constant" "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/relaykit/types" "github.com/QuantumNous/new-api/setting/operation_setting" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestOaiResponsesHandlerBuffersUnexpectedSSEAndPreservesUsage(t *testing.T) { terminal := `{"id":"resp_test","object":"response","model":"deepseek-v4.1-flash","status":"completed","extra_provider_field":true,"output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"OK"}]}],"usage":{"input_tokens":86926,"output_tokens":163,"total_tokens":87089,"input_tokens_details":{"cached_tokens":86784}}}` for _, tc := range []struct { name string body string wantError bool }{ {"completed", ": ping\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":" + terminal + "}\n\ndata: [DONE]\n\n", false}, {"incomplete", "data: {\"type\":\"response.incomplete\",\"response\":" + strings.Replace(terminal, `"completed"`, `"incomplete"`, 1) + "}\n\n", false}, {"truncated", "data: {\"type\":\"response.created\",\"response\":{\"model\":\"deepseek-v4.1-flash\"}}\n\ndata: [DONE]\n\n", true}, {"failed", "data: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\"}}\n\n", true}, {"invalid", "data: not-json\n\n", true}, } { t.Run(tc.name, func(t *testing.T) { w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{OriginModelName: "deepseek-v4.1-flash", ChannelMeta: &relaycommon.ChannelMeta{}} resp := &http.Response{StatusCode: 200, Header: http.Header{"Content-Type": {"text/event-stream; charset=utf-8"}}, Body: io.NopCloser(strings.NewReader(tc.body))} usage, apiErr := OaiResponsesHandler(c, info, resp) if tc.wantError { require.NotNil(t, apiErr) assert.Nil(t, usage) assert.False(t, c.Writer.Written()) return } require.Nil(t, apiErr) require.NotNil(t, usage) assert.Equal(t, 86926, usage.PromptTokens) assert.Equal(t, 163, usage.CompletionTokens) assert.Equal(t, 86784, usage.PromptTokensDetails.CachedTokens) assert.Equal(t, "application/json", w.Header().Get("Content-Type")) assert.Contains(t, w.Body.String(), `"extra_provider_field":true`) assert.Contains(t, w.Body.String(), `"text":"OK"`) }) } } func TestOaiResponsesHandlerCountsOutputCallsNotDeclarations(t *testing.T) { gin.SetMode(gin.TestMode) operation_setting.SetToolPriceForTest("priced_fn", 5.0) t.Cleanup(func() { operation_setting.DeleteToolPriceForTest("priced_fn") }) body, err := common.Marshal(dto.OpenAIResponsesResponse{ Tools: []map[string]any{ {"type": "web_search_preview"}, {"type": "file_search"}, }, Output: []dto.ResponsesOutput{ {Type: dto.BuildInCallWebSearchCall}, {Type: dto.BuildInCallWebSearchCall}, {Type: dto.BuildInCallFunctionCall, Name: "priced_fn"}, {Type: dto.BuildInCallFunctionCall, Name: "unpriced_fn"}, }, Usage: &dto.Usage{InputTokens: 1, OutputTokens: 1, TotalTokens: 2}, }) require.NoError(t, err) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{ OriginModelName: "gpt-5.1", ResponsesUsageInfo: &relaycommon.ResponsesUsageInfo{ BuiltInTools: map[string]*relaycommon.BuildInToolInfo{ dto.BuildInToolWebSearchPreview: {ToolName: dto.BuildInToolWebSearchPreview, CallCount: 0}, dto.BuildInToolFileSearch: {ToolName: dto.BuildInToolFileSearch, CallCount: 0}, }, }, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body)), Header: http.Header{"Content-Type": []string{"application/json"}}, } usage, apiErr := OaiResponsesHandler(c, info, resp) require.Nil(t, apiErr) require.NotNil(t, usage) assert.Equal(t, 2, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolWebSearchPreview].CallCount) assert.Equal(t, 0, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolFileSearch].CallCount) require.Contains(t, info.ResponsesUsageInfo.BuiltInTools, "priced_fn") assert.Equal(t, 1, info.ResponsesUsageInfo.BuiltInTools["priced_fn"].CallCount) assert.NotContains(t, info.ResponsesUsageInfo.BuiltInTools, "unpriced_fn") } func TestOaiResponsesHandlerDeclaredToolsWithoutOutputCountZero(t *testing.T) { gin.SetMode(gin.TestMode) body, err := common.Marshal(dto.OpenAIResponsesResponse{ Tools: []map[string]any{ {"type": "web_search_preview"}, {"type": "file_search"}, }, Output: []dto.ResponsesOutput{ {Type: "message", Role: "assistant"}, }, Usage: &dto.Usage{InputTokens: 1, OutputTokens: 1, TotalTokens: 2}, }) require.NoError(t, err) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{ OriginModelName: "gpt-5.1", ResponsesUsageInfo: &relaycommon.ResponsesUsageInfo{ BuiltInTools: map[string]*relaycommon.BuildInToolInfo{ dto.BuildInToolWebSearchPreview: {ToolName: dto.BuildInToolWebSearchPreview, CallCount: 0}, dto.BuildInToolFileSearch: {ToolName: dto.BuildInToolFileSearch, CallCount: 0}, }, }, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body)), Header: http.Header{"Content-Type": []string{"application/json"}}, } _, apiErr := OaiResponsesHandler(c, info, resp) require.Nil(t, apiErr) assert.Equal(t, 0, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolWebSearchPreview].CallCount) assert.Equal(t, 0, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolFileSearch].CallCount) } func TestOaiResponsesHandlerCountsCompletedImageGenerationOutputs(t *testing.T) { gin.SetMode(gin.TestMode) body, err := common.Marshal(dto.OpenAIResponsesResponse{ Status: []byte(`"completed"`), Output: []dto.ResponsesOutput{ { Type: dto.ResponsesOutputTypeImageGenerationCall, ID: "img_1", Status: "completed", Result: "base64-a", }, { Type: dto.ResponsesOutputTypeImageGenerationCall, ID: "img_2", Status: "completed", Result: "base64-b", }, { Type: dto.ResponsesOutputTypeImageGenerationCall, ID: "img_empty", Status: "completed", Result: "", }, }, Usage: &dto.Usage{InputTokens: 1, OutputTokens: 1, TotalTokens: 2}, }) require.NoError(t, err) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{OriginModelName: "gpt-5.1"} resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body)), Header: http.Header{"Content-Type": []string{"application/json"}}, } _, apiErr := OaiResponsesHandler(c, info, resp) require.Nil(t, apiErr) require.Contains(t, info.ResponsesUsageInfo.BuiltInTools, dto.BuildInToolImageGeneration) assert.Equal(t, 2, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolImageGeneration].CallCount) assert.False(t, c.GetBool("image_generation_call")) } func TestOaiResponsesHandlerCountsImageGenerationOnIncompleteStatus(t *testing.T) { gin.SetMode(gin.TestMode) body, err := common.Marshal(dto.OpenAIResponsesResponse{ Status: []byte(`"incomplete"`), Output: []dto.ResponsesOutput{ { Type: dto.ResponsesOutputTypeImageGenerationCall, ID: "img_1", Status: "completed", Result: "base64-a", }, }, Usage: &dto.Usage{InputTokens: 1, OutputTokens: 1, TotalTokens: 2}, }) require.NoError(t, err) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{ OriginModelName: "gpt-5.1", ResponsesUsageInfo: &relaycommon.ResponsesUsageInfo{ BuiltInTools: map[string]*relaycommon.BuildInToolInfo{ dto.BuildInToolImageGeneration: {ToolName: dto.BuildInToolImageGeneration, CallCount: 0}, }, }, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body)), Header: http.Header{"Content-Type": []string{"application/json"}}, } _, apiErr := OaiResponsesHandler(c, info, resp) require.Nil(t, apiErr) assert.Equal(t, 1, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolImageGeneration].CallCount) } func runResponsesImageBillingStream(t *testing.T, events ...string) *relaycommon.RelayInfo { t.Helper() gin.SetMode(gin.TestMode) oldTimeout := constant.StreamingTimeout constant.StreamingTimeout = 30 t.Cleanup(func() { constant.StreamingTimeout = oldTimeout }) var body strings.Builder for _, event := range events { body.WriteString("data: ") body.WriteString(event) body.WriteString("\n\n") } body.WriteString("data: [DONE]\n\n") w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) c.Set(common.RequestIdKey, "responses-image-billing-test") info := &relaycommon.RelayInfo{ OriginModelName: "gpt-5.1", DisablePing: true, ChannelMeta: &relaycommon.ChannelMeta{ UpstreamModelName: "gpt-5.1", }, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body.String())), Header: http.Header{"Content-Type": []string{"text/event-stream"}}, } _, apiErr := OaiResponsesStreamHandler(c, info, resp) require.Nil(t, apiErr) require.NotNil(t, info.ResponsesUsageInfo) require.Contains(t, info.ResponsesUsageInfo.BuiltInTools, dto.BuildInToolImageGeneration) return info } func TestOaiResponsesStreamHandlerDeduplicatesCompletedImageOutput(t *testing.T) { item := `{"type":"image_generation_call","id":"img_1","call_id":"call_1","status":"completed","result":"base64-a"}` info := runResponsesImageBillingStream( t, `{"type":"response.output_item.done","output_index":0,"item":`+item+`}`, `{"type":"response.completed","response":{"status":"completed","output":[`+item+`],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}`, ) assert.Equal(t, 1, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolImageGeneration].CallCount) } func TestOaiResponsesStreamHandlerCountsImageOutputOnIncomplete(t *testing.T) { info := runResponsesImageBillingStream( t, `{"type":"response.output_item.done","output_index":0,"item":{"type":"image_generation_call","id":"img_1","status":"completed","result":"base64-a"}}`, `{"type":"response.incomplete","response":{"status":"incomplete"}}`, ) assert.Equal(t, 1, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolImageGeneration].CallCount) } func TestOaiResponsesStreamHandlerDoesNotCountPartialImageEvent(t *testing.T) { info := runResponsesImageBillingStream( t, `{"type":"response.image_generation_call.partial_image","output_index":0,"partial_image_b64":"partial-bytes"}`, `{"type":"response.completed","response":{"status":"completed","output":[]}}`, ) assert.Equal(t, 0, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolImageGeneration].CallCount) } func TestOaiResponsesHandlerRewritesSGLangCreatedAtToInt(t *testing.T) { gin.SetMode(gin.TestMode) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{ ChannelMeta: &relaycommon.ChannelMeta{ChannelType: constant.ChannelTypeSGLang}, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"id":"resp_1","object":"response","created_at":1786588600.0,"status":"completed","output":[]}`)), Header: http.Header{"Content-Type": []string{"application/json"}}, } _, apiErr := OaiResponsesHandler(c, info, resp) require.Nil(t, apiErr) assert.Contains(t, w.Body.String(), `"created_at":1786588600`) assert.NotContains(t, w.Body.String(), `1786588600.0`) } func TestOaiResponsesStreamHandlerRewritesSGLangCreatedAtToInt(t *testing.T) { gin.SetMode(gin.TestMode) oldTimeout := constant.StreamingTimeout constant.StreamingTimeout = 30 t.Cleanup(func() { constant.StreamingTimeout = oldTimeout }) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) c.Set(common.RequestIdKey, "sglang-created-at-test") info := &relaycommon.RelayInfo{ DisablePing: true, ChannelMeta: &relaycommon.ChannelMeta{ ChannelType: constant.ChannelTypeSGLang, UpstreamModelName: "served-model", }, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader( "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"created_at\":1.7865886E9,\"status\":\"completed\",\"output\":[]}}\n\n" + "data: [DONE]\n\n", )), Header: http.Header{"Content-Type": []string{"text/event-stream"}}, } _, apiErr := OaiResponsesStreamHandler(c, info, resp) require.Nil(t, apiErr) assert.Contains(t, w.Body.String(), `"created_at":1786588600`) assert.NotContains(t, w.Body.String(), `1.7865886E9`) assert.NotContains(t, w.Body.String(), `1786588600.0`) } func TestOaiResponsesStreamHandlerKeepsNonSGLangCreatedAt(t *testing.T) { gin.SetMode(gin.TestMode) oldTimeout := constant.StreamingTimeout constant.StreamingTimeout = 30 t.Cleanup(func() { constant.StreamingTimeout = oldTimeout }) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) c.Set(common.RequestIdKey, "openai-created-at-test") info := &relaycommon.RelayInfo{ DisablePing: true, ChannelMeta: &relaycommon.ChannelMeta{ChannelType: constant.ChannelTypeOpenAI, UpstreamModelName: "gpt-test"}, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader( "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"created_at\":1786588600.0,\"status\":\"completed\",\"output\":[]}}\n\n" + "data: [DONE]\n\n", )), Header: http.Header{"Content-Type": []string{"text/event-stream"}}, } _, apiErr := OaiResponsesStreamHandler(c, info, resp) require.Nil(t, apiErr) assert.Contains(t, w.Body.String(), `1786588600.0`) } func TestResponsesStreamFailureHidesRoutingDetails(t *testing.T) { for _, eventType := range []string{"error", "response.error", "response.failed"} { t.Run(eventType, func(t *testing.T) { secret := "智能路由流式响应中途失败。原因:个人池商家在流式响应开始后中断。 request_id: upstream-private-id" original := map[string]any{"type": eventType, "sequence_number": 3, "code": "vendor_stream_error", "message": secret, "debug": secret, "error": map[string]any{"message": secret}, "response": map[string]any{"id": "resp_test", "status": "failed", "error": map[string]any{"type": "vendor_error", "code": "vendor_stream_error", "message": secret}, "usage": map[string]any{"input_tokens": 10, "output_tokens": 2}, "debug": secret}} encoded, err := common.Marshal(original) require.NoError(t, err) var event dto.ResponsesStreamResponse require.NoError(t, common.Unmarshal(encoded, &event)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) c.Set(common.RequestIdKey, "local-request-id") sendResponsesStreamData(c, event, string(encoded)) body := w.Body.String() assert.Contains(t, body, "event: "+eventType) assert.Contains(t, body, "model_service_error") assert.Contains(t, body, "local-request-id") assert.Contains(t, body, `"status":"failed"`) assert.Contains(t, body, `"input_tokens":10`) for _, detail := range []string{"个人池", "智能路由", "upstream-private-id", "vendor_stream_error", "debug"} { assert.NotContains(t, body, detail) } // Projection must not change the event used for outcome and billing. require.NotNil(t, event.Response) assert.Equal(t, "vendor_stream_error", event.Response.GetOpenAIError().Code) }) } } func TestResponsesSuccessfulStreamDataUnchanged(t *testing.T) { body := `{"type":"response.output_text.delta","delta":"个人池 is user content"}` var event dto.ResponsesStreamResponse require.NoError(t, common.UnmarshalJsonStr(body, &event)) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) sendResponsesStreamData(c, event, body) assert.Contains(t, w.Body.String(), body) } // A converted hosted web search reaches Azure and Xinference Chat Completions // as web_search_options, like OpenAI; the upstream decides whether it applies. // Azure Responses bills web search by tool_usage.web_search.num_requests (Bing // transactions) under bing_web_search, and the explicit zeros silence the // web_search_call items, which leaves one surcharge row. func TestAzureHostedWebSearch(t *testing.T) { gin.SetMode(gin.TestMode) oldTimeout := constant.StreamingTimeout constant.StreamingTimeout = 30 t.Cleanup(func() { constant.StreamingTimeout = oldTimeout }) t.Run("Claude to Chat", func(t *testing.T) { const body = `{"model":"gpt-5.1","max_tokens":64,"messages":[{"role":"user","content":"Weather?"}],"tools":[{"type":"web_search_20250305","name":"web_search"}]}` for _, channelType := range []int{constant.ChannelTypeAzure, constant.ChannelTypeXinference, constant.ChannelTypeOpenAI} { c, _ := gin.CreateTestContext(httptest.NewRecorder()) info := &relaycommon.RelayInfo{ RelayFormat: types.RelayFormatClaude, ChannelMeta: &relaycommon.ChannelMeta{ChannelType: channelType, UpstreamModelName: "gpt-5.1"}, } adaptor := &Adaptor{} adaptor.Init(info) var request dto.ClaudeRequest require.NoError(t, common.UnmarshalJsonStr(body, &request)) converted, err := adaptor.ConvertClaudeRequest(c, info, &request) require.NoError(t, err) wire, err := common.Marshal(converted) require.NoError(t, err) assert.True(t, strings.Contains(string(wire), `"web_search_options"`), channelType) var codes []string for _, diagnostic := range info.ConversionDiagnostics() { codes = append(codes, diagnostic.Code) } assert.False(t, slices.Contains(codes, "hosted_web_search_omitted"), channelType) } }) const searchCall = `{"type":"web_search_call","status":"completed","action":{"type":"search","query":"go"}}` const usage = `"usage":{"input_tokens":10,"output_tokens":3,"total_tokens":13},"tool_usage":{"web_search":{"num_requests":5}}` output := strings.Repeat(searchCall+",", 3) + `{"type":"message","role":"assistant","content":[{"type":"output_text","text":"Go 1.25"}]}` tests := []struct { name string contentType string body string }{ { name: "json", contentType: "application/json", body: `{"id":"resp_1","object":"response","model":"gpt-5.1","status":"completed","output":[` + output + `],` + usage + `}`, }, { name: "stream", contentType: "text/event-stream", body: strings.Repeat(`data: {"type":"response.output_item.done","output_index":0,"item":`+searchCall+"}\n\n", 3) + `data: {"type":"response.completed","response":{"id":"resp_1","object":"response","model":"gpt-5.1","status":"completed","output":[` + output + `],` + usage + "}}\n\ndata: [DONE]\n\n", }, } for _, tt := range tests { t.Run("Responses "+tt.name, func(t *testing.T) { c, _ := gin.CreateTestContext(httptest.NewRecorder()) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{ RelayMode: relayconstant.RelayModeResponses, RelayFormat: types.RelayFormatOpenAIResponses, OriginModelName: "gpt-5.1", IsStream: tt.name == "stream", DisablePing: true, ChannelMeta: &relaycommon.ChannelMeta{ChannelType: constant.ChannelTypeAzure, UpstreamModelName: "gpt-5.1"}, } adaptor := &Adaptor{} adaptor.Init(info) resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(tt.body)), Header: http.Header{"Content-Type": []string{tt.contentType}}, } var apiErr *types.NewAPIError if info.IsStream { _, apiErr = OaiResponsesStreamHandler(c, info, resp) } else { _, apiErr = OaiResponsesHandler(c, info, resp) } require.Nil(t, apiErr) billed := map[string]int{} for name, tool := range info.ResponsesUsageInfo.BuiltInTools { if tool.CallCount > 0 { billed[name] = tool.CallCount } } assert.Equal(t, map[string]int{"bing_web_search": 5}, billed) }) } }