feat: publish sanitized custom.97 source

relaykit/relayconvert/internal/claude_messages/to_oai_chat_req.go

@@ -7,8 +7,10 @@ import (
 
 	"github.com/QuantumNous/new-api/relaykit/dto"
 	"github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta"
+	"github.com/QuantumNous/new-api/relaykit/relayconvert/internal/convdiag"
 	kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil"
 	"github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning"
+	"github.com/QuantumNous/new-api/relaykit/types"
 )
 
 const (
@@ -154,7 +156,13 @@ func ClaudeMessagesRequestToOpenAIChat(ctx context.Context, claudeRequest dto.Cl
 		}
 	}
 
-	for _, claudeMessage := range claudeRequest.Messages {
+	type unnamedToolResult struct {
+		index int
+		id    string
+	}
+	toolNames := make(map[string]string)
+	var unnamedToolResults []unnamedToolResult
+	for messageIndex, claudeMessage := range claudeRequest.Messages {
 		openAIMessage := dto.Message{
 			Role: claudeMessage.Role,
 		}
@@ -168,7 +176,10 @@ func ClaudeMessagesRequestToOpenAIChat(ctx context.Context, claudeRequest dto.Cl
 			var toolCalls []dto.ToolCallRequest
 			mediaMessages := make([]dto.MediaContent, 0, len(content))
 
-			for _, mediaMsg := range content {
+			for blockIndex, mediaMsg := range content {
+				if _, exists := toolNames[mediaMsg.Id]; !exists {
+					toolNames[mediaMsg.Id] = mediaMsg.Name
+				}
 				switch mediaMsg.Type {
 				case "text", "input_text":
 					message := dto.MediaContent{
@@ -178,12 +189,50 @@ func ClaudeMessagesRequestToOpenAIChat(ctx context.Context, claudeRequest dto.Cl
 					}
 					mediaMessages = append(mediaMessages, message)
 				case "image":
-					imageData := fmt.Sprintf("data:%s;base64,%s", mediaMsg.Source.MediaType, mediaMsg.Source.Data)
-					mediaMessage := dto.MediaContent{
-						Type:     "image_url",
-						ImageUrl: &dto.MessageImageUrl{Url: imageData},
+					url := claudeImageSourceURL(mediaMsg.Source)
+					if url == "" {
+						sourceType := ""
+						if mediaMsg.Source != nil {
+							sourceType = mediaMsg.Source.Type
+						}
+						convdiag.Add(ctx, types.ConversionDiagnostic{
+							Code:     "unsupported_media_source",
+							Path:     fmt.Sprintf("messages[%d].content[%d].source", messageIndex, blockIndex),
+							Message:  fmt.Sprintf("OpenAI Chat Completions cannot reference a Claude image with source type %q; the image was omitted", sourceType),
+							Severity: types.ConversionDiagnosticError,
+						})
+						continue
 					}
-					mediaMessages = append(mediaMessages, mediaMessage)
+					mediaMessages = append(mediaMessages, dto.MediaContent{
+						Type:     "image_url",
+						ImageUrl: &dto.MessageImageUrl{Url: url},
+					})
+				case "document":
+					if mediaMsg.Source != nil && mediaMsg.Source.Type == "text" {
+						// A plain-text document keeps its content as a Chat text part.
+						mediaMessages = append(mediaMessages, dto.MediaContent{Type: "text", Text: kitutil.Interface2String(mediaMsg.Source.Data)})
+						continue
+					}
+					if mediaMsg.Source == nil || mediaMsg.Source.Type != "base64" || mediaMsg.Source.MediaType != "application/pdf" {
+						sourceType, mediaType := "", ""
+						if mediaMsg.Source != nil {
+							sourceType, mediaType = mediaMsg.Source.Type, mediaMsg.Source.MediaType
+						}
+						convdiag.Add(ctx, types.ConversionDiagnostic{
+							Code:     "unsupported_media_source",
+							Path:     fmt.Sprintf("messages[%d].content[%d].source", messageIndex, blockIndex),
+							Message:  fmt.Sprintf("OpenAI Chat Completions carries only base64 PDF and plain-text documents, not a %q source of type %q; the document was omitted", sourceType, mediaType),
+							Severity: types.ConversionDiagnosticError,
+						})
+						continue
+					}
+					mediaMessages = append(mediaMessages, dto.MediaContent{
+						Type: dto.ContentTypeFile,
+						File: &dto.MessageFile{
+							FileName: "document.pdf",
+							FileData: "data:application/pdf;base64," + kitutil.Interface2String(mediaMsg.Source.Data),
+						},
+					})
 				case "tool_use":
 					toolCall := dto.ToolCallRequest{
 						ID:   mediaMsg.Id,
@@ -197,7 +246,7 @@ func ClaudeMessagesRequestToOpenAIChat(ctx context.Context, claudeRequest dto.Cl
 				case "tool_result":
 					toolName := mediaMsg.Name
 					if toolName == "" {
-						toolName = claudeRequest.SearchToolNameByToolCallId(mediaMsg.ToolUseId)
+						unnamedToolResults = append(unnamedToolResults, unnamedToolResult{index: len(openAIMessages), id: mediaMsg.ToolUseId})
 					}
 					oaiToolMessage := dto.Message{
 						Role:       "tool",
@@ -207,9 +256,12 @@ func ClaudeMessagesRequestToOpenAIChat(ctx context.Context, claudeRequest dto.Cl
 					if mediaMsg.IsStringContent() {
 						oaiToolMessage.SetStringContent(mediaMsg.GetStringContent())
 					} else {
-						mediaContents := mediaMsg.ParseMediaContent()
-						encodedJSON, _ := kitutil.Marshal(mediaContents)
-						oaiToolMessage.SetStringContent(string(encodedJSON))
+						content, media := claudeToolResultToChat(mediaMsg.ParseMediaContent())
+						oaiToolMessage.SetStringContent(content)
+						// A Chat tool message only carries text. Images from the tool result
+						// join the user message this Claude message becomes, which lands right
+						// after the tool batch and keeps the tool messages contiguous.
+						mediaMessages = append(mediaMessages, media...)
 					}
 					openAIMessages = append(openAIMessages, oaiToolMessage)
 				}
@@ -218,7 +270,7 @@ func ClaudeMessagesRequestToOpenAIChat(ctx context.Context, claudeRequest dto.Cl
 			if len(toolCalls) > 0 {
 				openAIMessage.SetToolCalls(toolCalls)
 			}
-			if len(mediaMessages) > 0 && len(toolCalls) == 0 {
+			if len(mediaMessages) > 0 {
 				openAIMessage.SetMediaContent(mediaMessages)
 			}
 		}
@@ -226,11 +278,66 @@ func ClaudeMessagesRequestToOpenAIChat(ctx context.Context, claudeRequest dto.Cl
 			openAIMessages = append(openAIMessages, openAIMessage)
 		}
 	}
+	for _, result := range unnamedToolResults {
+		*openAIMessages[result.index].Name = toolNames[result.id]
+	}
 
 	openAIRequest.Messages = openAIMessages
 	return &openAIRequest, nil
 }
 
+// claudeToolResultToChat maps a structured tool_result content array onto Chat Completions,
+// where a tool message may only carry text. Text blocks stay on the tool message and image
+// blocks come back in Chat shape for the caller to place on the following user message;
+// stringifying them instead would hand base64 image data to the upstream text tokenizer.
+// Arrays holding any other block type keep the historical stringified form so nothing is lost.
+func claudeToolResultToChat(blocks []dto.ClaudeMediaMessage) (string, []dto.MediaContent) {
+	texts := make([]string, 0, len(blocks))
+	media := make([]dto.MediaContent, 0, len(blocks))
+	for _, block := range blocks {
+		switch {
+		case block.Type == "text" || block.Type == "input_text":
+			if text := block.GetText(); text != "" {
+				texts = append(texts, text)
+			}
+		case block.Type == "image":
+			url := claudeImageSourceURL(block.Source)
+			if url == "" {
+				return requestToJSONString(blocks), nil
+			}
+			media = append(media, dto.MediaContent{Type: "image_url", ImageUrl: &dto.MessageImageUrl{Url: url}})
+		default:
+			return requestToJSONString(blocks), nil
+		}
+	}
+	switch {
+	case len(texts) == 0 && len(media) == 0:
+		return requestToJSONString(blocks), nil
+	case len(texts) == 0:
+		// Upstreams reject empty tool content; the images ride on the following user message.
+		return "[image]", media
+	default:
+		return strings.Join(texts, "\n"), media
+	}
+}
+
+// claudeImageSourceURL is the Chat image_url for a Claude image source: a data
+// URL for base64 data or the URL of a url source. Other sources, such as Files
+// API references, have no Chat equivalent and yield "".
+func claudeImageSourceURL(source *dto.ClaudeMessageSource) string {
+	if source == nil || source.Type == "file" {
+		return ""
+	}
+	if source.Url != "" {
+		return source.Url
+	}
+	data := kitutil.Interface2String(source.Data)
+	if data == "" {
+		return ""
+	}
+	return fmt.Sprintf("data:%s;base64,%s", source.MediaType, data)
+}
+
 func requestToJSONString(v any) string {
 	b, err := kitutil.Marshal(v)
 	if err != nil {

relaykit/relayconvert/internal/claude_messages/to_oai_chat_req_test.go

@@ -0,0 +1,151 @@
+package claudemessages
+
+import (
+	"context"
+	"testing"
+
+	"github.com/QuantumNous/new-api/relaykit/dto"
+	"github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta"
+	"github.com/samber/lo"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClaudeMessagesRequestToOpenAIChatToolResultContent(t *testing.T) {
+	const imageData = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg=="
+	const dataURL = "data:image/png;base64," + imageData
+	imageBlock := dto.ClaudeMediaMessage{
+		Type:   "image",
+		Source: &dto.ClaudeMessageSource{Type: "base64", MediaType: "image/png", Data: imageData},
+	}
+	textBlock := func(text string) dto.ClaudeMediaMessage {
+		return dto.ClaudeMediaMessage{Type: "text", Text: lo.ToPtr(text)}
+	}
+
+	tests := []struct {
+		name        string
+		content     any
+		wantContent string
+		wantImages  []string
+	}{
+		{
+			name:        "string passes through",
+			content:     "done",
+			wantContent: "done",
+		},
+		{
+			name:        "text and image keep text on tool message",
+			content:     []dto.ClaudeMediaMessage{textBlock("screenshot taken"), imageBlock},
+			wantContent: "screenshot taken",
+			wantImages:  []string{dataURL},
+		},
+		{
+			name:        "image only uses placeholder",
+			content:     []dto.ClaudeMediaMessage{imageBlock},
+			wantContent: "[image]",
+			wantImages:  []string{dataURL},
+		},
+		{
+			name: "url image source is forwarded as is",
+			content: []dto.ClaudeMediaMessage{{
+				Type:   "image",
+				Source: &dto.ClaudeMessageSource{Type: "url", Url: "https://example.com/shot.png"},
+			}},
+			wantContent: "[image]",
+			wantImages:  []string{"https://example.com/shot.png"},
+		},
+		{
+			name:        "text blocks join with newline",
+			content:     []dto.ClaudeMediaMessage{textBlock("first"), textBlock(""), textBlock("second")},
+			wantContent: "first\nsecond",
+		},
+		{
+			name:        "unknown block keeps whole array",
+			content:     []dto.ClaudeMediaMessage{textBlock("see doc"), {Type: "document", Data: "opaque"}},
+			wantContent: `[{"type":"text","text":"see doc"},{"type":"document","data":"opaque"}]`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got, err := ClaudeMessagesRequestToOpenAIChat(context.Background(), dto.ClaudeRequest{
+				Model:     "claude-test",
+				MaxTokens: lo.ToPtr(uint(64)),
+				Messages: []dto.ClaudeMessage{
+					{Role: "user", Content: "take a screenshot"},
+					{Role: "assistant", Content: []dto.ClaudeMediaMessage{
+						{Type: "tool_use", Id: "toolu_1", Name: "screenshot", Input: map[string]any{}},
+					}},
+					{Role: "user", Content: []dto.ClaudeMediaMessage{
+						{Type: "tool_result", ToolUseId: "toolu_1", Content: tt.content},
+						textBlock("what do you see?"),
+					}},
+				},
+			}, &convmeta.Values{})
+			require.NoError(t, err)
+
+			require.Len(t, got.Messages, 4)
+			tool := got.Messages[2]
+			assert.Equal(t, "tool", tool.Role)
+			assert.Equal(t, "toolu_1", tool.ToolCallId)
+			if tt.wantImages == nil && tt.wantContent[0] == '[' {
+				assert.JSONEq(t, tt.wantContent, tool.StringContent())
+			} else {
+				assert.Equal(t, tt.wantContent, tool.StringContent())
+			}
+
+			user := got.Messages[3]
+			assert.Equal(t, "user", user.Role)
+			parts := user.ParseContent()
+			require.Len(t, parts, len(tt.wantImages)+1)
+			for i, wantURL := range tt.wantImages {
+				assert.Equal(t, dto.ContentTypeImageURL, parts[i].Type)
+				require.NotNil(t, parts[i].GetImageMedia())
+				assert.Equal(t, wantURL, parts[i].GetImageMedia().Url)
+			}
+			assert.Equal(t, dto.ContentTypeText, parts[len(parts)-1].Type)
+			assert.Equal(t, "what do you see?", parts[len(parts)-1].Text)
+		})
+	}
+}
+
+func TestClaudeMessagesRequestToOpenAIChatKeepsParallelToolResultsContiguous(t *testing.T) {
+	const dataURL = "data:image/png;base64,iVBORw0KGgo="
+	got, err := ClaudeMessagesRequestToOpenAIChat(context.Background(), dto.ClaudeRequest{
+		Model:     "claude-test",
+		MaxTokens: lo.ToPtr(uint(64)),
+		Messages: []dto.ClaudeMessage{
+			{Role: "user", Content: "screenshot and read the note"},
+			{Role: "assistant", Content: []dto.ClaudeMediaMessage{
+				{Type: "text", Text: lo.ToPtr("Let me look.")},
+				{Type: "tool_use", Id: "toolu_shot", Name: "screenshot", Input: map[string]any{}},
+				{Type: "tool_use", Id: "toolu_read", Name: "read_file", Input: map[string]any{"path": "note.txt"}},
+			}},
+			{Role: "user", Content: []dto.ClaudeMediaMessage{
+				{Type: "tool_result", ToolUseId: "toolu_shot", Content: []dto.ClaudeMediaMessage{{
+					Type:   "image",
+					Source: &dto.ClaudeMessageSource{Type: "base64", MediaType: "image/png", Data: "iVBORw0KGgo="},
+				}}},
+				{Type: "tool_result", ToolUseId: "toolu_read", Content: "note contents"},
+			}},
+		},
+	}, &convmeta.Values{})
+	require.NoError(t, err)
+
+	roles := make([]string, 0, len(got.Messages))
+	for _, message := range got.Messages {
+		roles = append(roles, message.Role)
+	}
+	assert.Equal(t, []string{"user", "assistant", "tool", "tool", "user"}, roles)
+	assert.Len(t, got.Messages[1].ParseToolCalls(), 2)
+	assert.Equal(t, "[image]", got.Messages[2].StringContent())
+	assert.Equal(t, "toolu_shot", got.Messages[2].ToolCallId)
+	assert.Equal(t, "note contents", got.Messages[3].StringContent())
+	assert.NotContains(t, got.Messages[2].StringContent()+got.Messages[3].StringContent(), "base64")
+
+	parts := got.Messages[4].ParseContent()
+	require.Len(t, parts, 1)
+	assert.Equal(t, dto.ContentTypeImageURL, parts[0].Type)
+	require.NotNil(t, parts[0].GetImageMedia())
+	assert.Equal(t, dataURL, parts[0].GetImageMedia().Url)
+}

relaykit/relayconvert/internal/claude_messages/to_oai_chat_resp.go

@@ -19,6 +19,8 @@ type ClaudeResponseInfo struct {
 	Created      int64
 	Model        string
 	ResponseText strings.Builder
+	// ToolUseCount counts streamed tool_use blocks for the output-token fallback.
+	ToolUseCount int
 	Usage        *dto.Usage
 	Done         bool
 
@@ -524,6 +526,9 @@ func FormatClaudeResponseInfo(claudeResponse *dto.ClaudeResponse, oaiResponse *d
 			if claudeResponse.Delta.Thinking != nil {
 				claudeInfo.ResponseText.WriteString(*claudeResponse.Delta.Thinking)
 			}
+			if claudeResponse.Delta.PartialJson != nil {
+				claudeInfo.ResponseText.WriteString(*claudeResponse.Delta.PartialJson)
+			}
 		}
 	} else if claudeResponse.Type == "message_delta" {
 		if claudeResponse.Usage != nil {
@@ -552,6 +557,11 @@ func FormatClaudeResponseInfo(claudeResponse *dto.ClaudeResponse, oaiResponse *d
 
 		claudeInfo.Done = true
 	} else if claudeResponse.Type == "content_block_start" {
+		// Tool calls are most of an agent turn's output; count them for usage fallback.
+		if block := claudeResponse.ContentBlock; block != nil && block.Type == "tool_use" {
+			claudeInfo.ResponseText.WriteString(block.Name)
+			claudeInfo.ToolUseCount++
+		}
 	} else {
 		return false
 	}

relaykit/relayconvert/internal/claude_messages/to_oai_responses_req.go

@@ -10,6 +10,7 @@ import (
 	"github.com/QuantumNous/new-api/relaykit/dto"
 	"github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta"
 	"github.com/QuantumNous/new-api/relaykit/relayconvert/internal/convdiag"
+	relaymedia "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/media"
 	kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil"
 	"github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning"
 )
@@ -177,6 +178,7 @@ func claudeMessagesToResponsesInput(messages []dto.ClaudeMessage) (json.RawMessa
 					contentParts = append(contentParts, map[string]any{
 						"type":      "input_image",
 						"image_url": source,
+						"detail":    relaymedia.DefaultResponsesImageDetail,
 					})
 				}
 			case "document":
@@ -298,7 +300,11 @@ func claudeToolResultToResponsesOutput(content any) (any, error) {
 			parts = append(parts, map[string]any{"type": "input_text", "text": block.GetText()})
 		case "image":
 			if source := claudeSourceURL(block.Source); source != "" {
-				parts = append(parts, map[string]any{"type": "input_image", "image_url": source})
+				parts = append(parts, map[string]any{
+					"type":      "input_image",
+					"image_url": source,
+					"detail":    relaymedia.DefaultResponsesImageDetail,
+				})
 			}
 		case "document":
 			if source := claudeSourceURL(block.Source); source != "" {