package openai import ( "bytes" "io" "net/http" "net/http/httptest" "strings" "testing" "github.com/QuantumNous/new-api/common" "github.com/QuantumNous/new-api/constant" relaycommon "github.com/QuantumNous/new-api/relay/common" "github.com/QuantumNous/new-api/relaykit/dto" "github.com/QuantumNous/new-api/setting/operation_setting" "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) func TestOaiResponsesHandlerBuffersUnexpectedSSEAndPreservesUsage(t *testing.T) { terminal := `{"id":"resp_test","object":"response","model":"deepseek-v4.1-flash","status":"completed","extra_provider_field":true,"output":[{"type":"message","role":"assistant","content":[{"type":"output_text","text":"OK"}]}],"usage":{"input_tokens":86926,"output_tokens":163,"total_tokens":87089,"input_tokens_details":{"cached_tokens":86784}}}` for _, tc := range []struct { name string body string wantError bool }{ {"completed", ": ping\n\nevent: response.completed\ndata: {\"type\":\"response.completed\",\"response\":" + terminal + "}\n\ndata: [DONE]\n\n", false}, {"incomplete", "data: {\"type\":\"response.incomplete\",\"response\":" + strings.Replace(terminal, `"completed"`, `"incomplete"`, 1) + "}\n\n", false}, {"truncated", "data: {\"type\":\"response.created\",\"response\":{\"model\":\"deepseek-v4.1-flash\"}}\n\ndata: [DONE]\n\n", true}, {"failed", "data: {\"type\":\"response.failed\",\"response\":{\"status\":\"failed\"}}\n\n", true}, {"invalid", "data: not-json\n\n", true}, } { t.Run(tc.name, func(t *testing.T) { w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{OriginModelName: "deepseek-v4.1-flash", ChannelMeta: &relaycommon.ChannelMeta{}} resp := &http.Response{StatusCode: 200, Header: http.Header{"Content-Type": {"text/event-stream; charset=utf-8"}}, Body: io.NopCloser(strings.NewReader(tc.body))} usage, apiErr := OaiResponsesHandler(c, info, resp) if tc.wantError { require.NotNil(t, apiErr) assert.Nil(t, usage) assert.False(t, c.Writer.Written()) return } require.Nil(t, apiErr) require.NotNil(t, usage) assert.Equal(t, 86926, usage.PromptTokens) assert.Equal(t, 163, usage.CompletionTokens) assert.Equal(t, 86784, usage.PromptTokensDetails.CachedTokens) assert.Equal(t, "application/json", w.Header().Get("Content-Type")) assert.Contains(t, w.Body.String(), `"extra_provider_field":true`) assert.Contains(t, w.Body.String(), `"text":"OK"`) }) } } func TestOaiResponsesHandlerCountsOutputCallsNotDeclarations(t *testing.T) { gin.SetMode(gin.TestMode) operation_setting.SetToolPriceForTest("priced_fn", 5.0) t.Cleanup(func() { operation_setting.DeleteToolPriceForTest("priced_fn") }) body, err := common.Marshal(dto.OpenAIResponsesResponse{ Tools: []map[string]any{ {"type": "web_search_preview"}, {"type": "file_search"}, }, Output: []dto.ResponsesOutput{ {Type: dto.BuildInCallWebSearchCall}, {Type: dto.BuildInCallWebSearchCall}, {Type: dto.BuildInCallFunctionCall, Name: "priced_fn"}, {Type: dto.BuildInCallFunctionCall, Name: "unpriced_fn"}, }, Usage: &dto.Usage{InputTokens: 1, OutputTokens: 1, TotalTokens: 2}, }) require.NoError(t, err) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{ OriginModelName: "gpt-5.1", ResponsesUsageInfo: &relaycommon.ResponsesUsageInfo{ BuiltInTools: map[string]*relaycommon.BuildInToolInfo{ dto.BuildInToolWebSearchPreview: {ToolName: dto.BuildInToolWebSearchPreview, CallCount: 0}, dto.BuildInToolFileSearch: {ToolName: dto.BuildInToolFileSearch, CallCount: 0}, }, }, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body)), Header: http.Header{"Content-Type": []string{"application/json"}}, } usage, apiErr := OaiResponsesHandler(c, info, resp) require.Nil(t, apiErr) require.NotNil(t, usage) assert.Equal(t, 2, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolWebSearchPreview].CallCount) assert.Equal(t, 0, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolFileSearch].CallCount) require.Contains(t, info.ResponsesUsageInfo.BuiltInTools, "priced_fn") assert.Equal(t, 1, info.ResponsesUsageInfo.BuiltInTools["priced_fn"].CallCount) assert.NotContains(t, info.ResponsesUsageInfo.BuiltInTools, "unpriced_fn") } func TestOaiResponsesHandlerDeclaredToolsWithoutOutputCountZero(t *testing.T) { gin.SetMode(gin.TestMode) body, err := common.Marshal(dto.OpenAIResponsesResponse{ Tools: []map[string]any{ {"type": "web_search_preview"}, {"type": "file_search"}, }, Output: []dto.ResponsesOutput{ {Type: "message", Role: "assistant"}, }, Usage: &dto.Usage{InputTokens: 1, OutputTokens: 1, TotalTokens: 2}, }) require.NoError(t, err) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{ OriginModelName: "gpt-5.1", ResponsesUsageInfo: &relaycommon.ResponsesUsageInfo{ BuiltInTools: map[string]*relaycommon.BuildInToolInfo{ dto.BuildInToolWebSearchPreview: {ToolName: dto.BuildInToolWebSearchPreview, CallCount: 0}, dto.BuildInToolFileSearch: {ToolName: dto.BuildInToolFileSearch, CallCount: 0}, }, }, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body)), Header: http.Header{"Content-Type": []string{"application/json"}}, } _, apiErr := OaiResponsesHandler(c, info, resp) require.Nil(t, apiErr) assert.Equal(t, 0, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolWebSearchPreview].CallCount) assert.Equal(t, 0, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolFileSearch].CallCount) } func TestOaiResponsesHandlerCountsCompletedImageGenerationOutputs(t *testing.T) { gin.SetMode(gin.TestMode) body, err := common.Marshal(dto.OpenAIResponsesResponse{ Status: []byte(`"completed"`), Output: []dto.ResponsesOutput{ { Type: dto.ResponsesOutputTypeImageGenerationCall, ID: "img_1", Status: "completed", Result: "base64-a", }, { Type: dto.ResponsesOutputTypeImageGenerationCall, ID: "img_2", Status: "completed", Result: "base64-b", }, { Type: dto.ResponsesOutputTypeImageGenerationCall, ID: "img_empty", Status: "completed", Result: "", }, }, Usage: &dto.Usage{InputTokens: 1, OutputTokens: 1, TotalTokens: 2}, }) require.NoError(t, err) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{OriginModelName: "gpt-5.1"} resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body)), Header: http.Header{"Content-Type": []string{"application/json"}}, } _, apiErr := OaiResponsesHandler(c, info, resp) require.Nil(t, apiErr) require.Contains(t, info.ResponsesUsageInfo.BuiltInTools, dto.BuildInToolImageGeneration) assert.Equal(t, 2, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolImageGeneration].CallCount) assert.False(t, c.GetBool("image_generation_call")) } func TestOaiResponsesHandlerIncompleteStatusCommitsZeroImageGeneration(t *testing.T) { gin.SetMode(gin.TestMode) body, err := common.Marshal(dto.OpenAIResponsesResponse{ Status: []byte(`"incomplete"`), Output: []dto.ResponsesOutput{ { Type: dto.ResponsesOutputTypeImageGenerationCall, ID: "img_1", Status: "completed", Result: "base64-a", }, }, Usage: &dto.Usage{InputTokens: 1, OutputTokens: 1, TotalTokens: 2}, }) require.NoError(t, err) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{ OriginModelName: "gpt-5.1", ResponsesUsageInfo: &relaycommon.ResponsesUsageInfo{ BuiltInTools: map[string]*relaycommon.BuildInToolInfo{ dto.BuildInToolImageGeneration: {ToolName: dto.BuildInToolImageGeneration, CallCount: 0}, }, }, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(bytes.NewReader(body)), Header: http.Header{"Content-Type": []string{"application/json"}}, } _, apiErr := OaiResponsesHandler(c, info, resp) require.Nil(t, apiErr) assert.Equal(t, 0, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolImageGeneration].CallCount) } func runResponsesImageBillingStream(t *testing.T, events ...string) *relaycommon.RelayInfo { t.Helper() gin.SetMode(gin.TestMode) oldTimeout := constant.StreamingTimeout constant.StreamingTimeout = 30 t.Cleanup(func() { constant.StreamingTimeout = oldTimeout }) var body strings.Builder for _, event := range events { body.WriteString("data: ") body.WriteString(event) body.WriteString("\n\n") } body.WriteString("data: [DONE]\n\n") w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) c.Set(common.RequestIdKey, "responses-image-billing-test") info := &relaycommon.RelayInfo{ OriginModelName: "gpt-5.1", DisablePing: true, ChannelMeta: &relaycommon.ChannelMeta{ UpstreamModelName: "gpt-5.1", }, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(body.String())), Header: http.Header{"Content-Type": []string{"text/event-stream"}}, } _, apiErr := OaiResponsesStreamHandler(c, info, resp) require.Nil(t, apiErr) require.NotNil(t, info.ResponsesUsageInfo) require.Contains(t, info.ResponsesUsageInfo.BuiltInTools, dto.BuildInToolImageGeneration) return info } func TestOaiResponsesStreamHandlerDeduplicatesCompletedImageOutput(t *testing.T) { item := `{"type":"image_generation_call","id":"img_1","call_id":"call_1","status":"completed","result":"base64-a"}` info := runResponsesImageBillingStream( t, `{"type":"response.output_item.done","output_index":0,"item":`+item+`}`, `{"type":"response.completed","response":{"status":"completed","output":[`+item+`],"usage":{"input_tokens":1,"output_tokens":1,"total_tokens":2}}}`, ) assert.Equal(t, 1, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolImageGeneration].CallCount) } func TestOaiResponsesStreamHandlerDiscardsImageOutputOnIncomplete(t *testing.T) { info := runResponsesImageBillingStream( t, `{"type":"response.output_item.done","output_index":0,"item":{"type":"image_generation_call","id":"img_1","status":"completed","result":"base64-a"}}`, `{"type":"response.incomplete","response":{"status":"incomplete"}}`, ) assert.Equal(t, 0, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolImageGeneration].CallCount) } func TestOaiResponsesStreamHandlerDoesNotCountPartialImageEvent(t *testing.T) { info := runResponsesImageBillingStream( t, `{"type":"response.image_generation_call.partial_image","output_index":0,"partial_image_b64":"partial-bytes"}`, `{"type":"response.completed","response":{"status":"completed","output":[]}}`, ) assert.Equal(t, 0, info.ResponsesUsageInfo.BuiltInTools[dto.BuildInToolImageGeneration].CallCount) } func TestOaiResponsesHandlerRewritesSGLangCreatedAtToInt(t *testing.T) { gin.SetMode(gin.TestMode) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) info := &relaycommon.RelayInfo{ ChannelMeta: &relaycommon.ChannelMeta{ChannelType: constant.ChannelTypeSGLang}, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader(`{"id":"resp_1","object":"response","created_at":1786588600.0,"status":"completed","output":[]}`)), Header: http.Header{"Content-Type": []string{"application/json"}}, } _, apiErr := OaiResponsesHandler(c, info, resp) require.Nil(t, apiErr) assert.Contains(t, w.Body.String(), `"created_at":1786588600`) assert.NotContains(t, w.Body.String(), `1786588600.0`) } func TestOaiResponsesStreamHandlerRewritesSGLangCreatedAtToInt(t *testing.T) { gin.SetMode(gin.TestMode) oldTimeout := constant.StreamingTimeout constant.StreamingTimeout = 30 t.Cleanup(func() { constant.StreamingTimeout = oldTimeout }) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) c.Set(common.RequestIdKey, "sglang-created-at-test") info := &relaycommon.RelayInfo{ DisablePing: true, ChannelMeta: &relaycommon.ChannelMeta{ ChannelType: constant.ChannelTypeSGLang, UpstreamModelName: "served-model", }, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader( "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"created_at\":1.7865886E9,\"status\":\"completed\",\"output\":[]}}\n\n" + "data: [DONE]\n\n", )), Header: http.Header{"Content-Type": []string{"text/event-stream"}}, } _, apiErr := OaiResponsesStreamHandler(c, info, resp) require.Nil(t, apiErr) assert.Contains(t, w.Body.String(), `"created_at":1786588600`) assert.NotContains(t, w.Body.String(), `1.7865886E9`) assert.NotContains(t, w.Body.String(), `1786588600.0`) } func TestOaiResponsesStreamHandlerKeepsNonSGLangCreatedAt(t *testing.T) { gin.SetMode(gin.TestMode) oldTimeout := constant.StreamingTimeout constant.StreamingTimeout = 30 t.Cleanup(func() { constant.StreamingTimeout = oldTimeout }) w := httptest.NewRecorder() c, _ := gin.CreateTestContext(w) c.Request = httptest.NewRequest(http.MethodPost, "/v1/responses", nil) c.Set(common.RequestIdKey, "openai-created-at-test") info := &relaycommon.RelayInfo{ DisablePing: true, ChannelMeta: &relaycommon.ChannelMeta{ChannelType: constant.ChannelTypeOpenAI, UpstreamModelName: "gpt-test"}, } resp := &http.Response{ StatusCode: http.StatusOK, Body: io.NopCloser(strings.NewReader( "data: {\"type\":\"response.completed\",\"response\":{\"id\":\"resp_1\",\"created_at\":1786588600.0,\"status\":\"completed\",\"output\":[]}}\n\n" + "data: [DONE]\n\n", )), Header: http.Header{"Content-Type": []string{"text/event-stream"}}, } _, apiErr := OaiResponsesStreamHandler(c, info, resp) require.Nil(t, apiErr) assert.Contains(t, w.Body.String(), `1786588600.0`) }