441 lines
15 KiB
Go
441 lines
15 KiB
Go
package oairesponses
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
|
|
"context"
|
|
|
|
"github.com/QuantumNous/new-api/relaykit/dto"
|
|
"github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta"
|
|
"github.com/QuantumNous/new-api/relaykit/relayconvert/internal/convdiag"
|
|
relaymedia "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/media"
|
|
sharedgemini "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/shared/gemini"
|
|
kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil"
|
|
"github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning"
|
|
"github.com/QuantumNous/new-api/relaykit/types"
|
|
)
|
|
|
|
func OpenAIResponsesRequestToGeminiChat(c context.Context, req *dto.OpenAIResponsesRequest, info convmeta.Meta) (*dto.GeminiChatRequest, error) {
|
|
opts := convmeta.OptionsOf(info)
|
|
if req == nil {
|
|
return nil, fmt.Errorf("request is nil")
|
|
}
|
|
if req.Model == "" {
|
|
return nil, fmt.Errorf("model is required")
|
|
}
|
|
if err := ValidateRequestChatUnsupportedFields(req); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
geminiRequest := &dto.GeminiChatRequest{
|
|
GenerationConfig: dto.GeminiChatGenerationConfig{
|
|
Temperature: req.Temperature,
|
|
},
|
|
}
|
|
if req.TopP != nil {
|
|
geminiRequest.GenerationConfig.TopP = kitutil.GetPointer(*req.TopP)
|
|
}
|
|
if req.MaxOutputTokens != nil {
|
|
geminiRequest.GenerationConfig.MaxOutputTokens = kitutil.GetPointer(*req.MaxOutputTokens)
|
|
}
|
|
|
|
upstreamModelName := req.Model
|
|
if modelName := convmeta.UpstreamModelName(info); modelName != "" {
|
|
upstreamModelName = modelName
|
|
}
|
|
if opts.Gemini.SupportsImagineModel(upstreamModelName) {
|
|
geminiRequest.GenerationConfig.ResponseModalities = []string{"TEXT", "IMAGE"}
|
|
}
|
|
if err := applyResponsesTextToGemini(c, req.Text, geminiRequest, opts.Gemini.SupportsJSONSchemaModel(upstreamModelName)); err != nil {
|
|
return nil, err
|
|
}
|
|
reasoningIntent, diagnostics, err := reasoning.FromOpenAIResponses(req)
|
|
if err != nil {
|
|
return nil, reasoning.AsClientError(err)
|
|
}
|
|
convdiag.Add(c, diagnostics...)
|
|
var reasoningPivot dto.GeneralOpenAIRequest
|
|
if err := reasoning.ApplyToOpenAIChat(&reasoningPivot, reasoningIntent); err != nil {
|
|
return nil, reasoning.AsClientError(err)
|
|
}
|
|
reasoningPivot.Model = req.Model
|
|
reasoningPivot.MaxCompletionTokens = req.MaxOutputTokens
|
|
if err := sharedgemini.ApplyThinkingConfig(c, geminiRequest, info, reasoningPivot); err != nil {
|
|
return nil, reasoning.AsClientError(err)
|
|
}
|
|
|
|
var safetySettings []dto.GeminiChatSafetySettings
|
|
for _, category := range sharedgemini.SafetySettingCategories {
|
|
threshold := opts.Gemini.SafetySettingFor(category)
|
|
if threshold == "" {
|
|
continue
|
|
}
|
|
safetySettings = append(safetySettings, dto.GeminiChatSafetySettings{
|
|
Category: category,
|
|
Threshold: threshold,
|
|
})
|
|
}
|
|
if len(safetySettings) > 0 {
|
|
geminiRequest.SafetySettings = safetySettings
|
|
}
|
|
|
|
functions, err := RequestFunctionDeclarations(req.Tools)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
for i := range functions {
|
|
if params, ok := functions[i].Parameters.(map[string]any); ok {
|
|
if props, hasProps := params["properties"].(map[string]any); hasProps && len(props) == 0 {
|
|
functions[i].Parameters = nil
|
|
continue
|
|
}
|
|
}
|
|
functions[i].Parameters = sharedgemini.CleanFunctionParameters(functions[i].Parameters)
|
|
}
|
|
if len(functions) > 0 {
|
|
geminiRequest.SetTools([]dto.GeminiChatTool{
|
|
{FunctionDeclarations: functions},
|
|
})
|
|
}
|
|
|
|
toolChoice, err := RequestToolChoiceToChat(req.ToolChoice)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if toolChoice != nil {
|
|
geminiRequest.ToolConfig = sharedgemini.OpenAIToolChoiceToConfig(toolChoice)
|
|
}
|
|
|
|
systemTexts := make([]string, 0)
|
|
if RawJSONPresent(req.Instructions) {
|
|
instructions, err := JSONString(req.Instructions)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("invalid instructions: %w", err)
|
|
}
|
|
if strings.TrimSpace(instructions) != "" {
|
|
systemTexts = append(systemTexts, instructions)
|
|
}
|
|
}
|
|
|
|
inputItems, err := InputItems(req.Input)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
// Multimodal function responses (media in functionResponse.parts) are
|
|
// documented for Gemini 3. Gemini 1.x and 2.x get tool media as ordinary
|
|
// inlineData parts after the function responses of the same turn.
|
|
modelID := strings.TrimPrefix(strings.ToLower(upstreamModelName), "models/")
|
|
toolMediaInFunctionResponse := !strings.HasPrefix(modelID, "gemini-1.") && !strings.HasPrefix(modelID, "gemini-2.")
|
|
callNames := make(map[string]string)
|
|
for _, item := range inputItems {
|
|
itemType := strings.TrimSpace(kitutil.Interface2String(item["type"]))
|
|
switch itemType {
|
|
case ResponsesInputTypeFunctionCall, ResponsesInputTypeCustomToolCall:
|
|
part, callID, err := responsesFunctionCallItemToGeminiPart(item, itemType)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
sharedgemini.AttachFunctionCallThoughtSignature(opts, &part)
|
|
if callID != "" {
|
|
callNames[callID] = part.FunctionCall.FunctionName
|
|
}
|
|
appendGeminiContentPart(geminiRequest, "model", part)
|
|
case ResponsesInputTypeFunctionCallOutput, ResponsesInputTypeCustomToolOutput:
|
|
part, media, err := responsesFunctionOutputItemToGeminiPart(c, item, callNames, toolMediaInFunctionResponse)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
appendGeminiContentPart(geminiRequest, "user", part)
|
|
turn := &geminiRequest.Contents[len(geminiRequest.Contents)-1]
|
|
turn.Parts = append(turn.Parts, media...)
|
|
default:
|
|
role := responsesGeminiRole(item)
|
|
parts, err := responsesInputContentToGeminiParts(c, item["content"])
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if role == "system" {
|
|
for _, part := range parts {
|
|
if part.Text != "" {
|
|
systemTexts = append(systemTexts, part.Text)
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
if len(parts) > 0 {
|
|
geminiRequest.Contents = append(geminiRequest.Contents, dto.GeminiChatContent{
|
|
Role: role,
|
|
Parts: parts,
|
|
})
|
|
}
|
|
}
|
|
}
|
|
geminiRequest.Contents = omitTrailingAssistantText(c, geminiRequest.Contents,
|
|
func(content dto.GeminiChatContent) bool { return content.Role == "user" },
|
|
func(content dto.GeminiChatContent) bool {
|
|
return content.Role == "model" && !slices.ContainsFunc(content.Parts, func(part dto.GeminiPart) bool { return part.FunctionCall != nil })
|
|
})
|
|
|
|
if len(systemTexts) > 0 {
|
|
geminiRequest.SystemInstructions = &dto.GeminiChatContent{
|
|
Parts: []dto.GeminiPart{{Text: strings.Join(systemTexts, "\n")}},
|
|
}
|
|
}
|
|
|
|
return geminiRequest, nil
|
|
}
|
|
|
|
func applyResponsesTextToGemini(c context.Context, raw []byte, geminiRequest *dto.GeminiChatRequest, jsonSchema bool) error {
|
|
responseFormat, err := RequestTextToChatResponseFormat(raw)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if responseFormat == nil || (responseFormat.Type != "json_schema" && responseFormat.Type != "json_object") {
|
|
return nil
|
|
}
|
|
|
|
geminiRequest.GenerationConfig.ResponseMimeType = "application/json"
|
|
if len(responseFormat.JsonSchema) == 0 {
|
|
return nil
|
|
}
|
|
|
|
sharedgemini.ApplyResponseSchema(c, &geminiRequest.GenerationConfig, responseFormat.JsonSchema, "text.format.schema", jsonSchema)
|
|
return nil
|
|
}
|
|
|
|
func responsesInputContentToGeminiParts(c context.Context, content any) ([]dto.GeminiPart, error) {
|
|
contentParts, err := ContentParts(content)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
parts := make([]dto.GeminiPart, 0, len(contentParts))
|
|
for _, contentPart := range contentParts {
|
|
nextParts, err := responsesContentPartToGeminiParts(c, contentPart)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
parts = append(parts, nextParts...)
|
|
}
|
|
return parts, nil
|
|
}
|
|
|
|
func responsesContentPartToGeminiParts(c context.Context, part map[string]any) ([]dto.GeminiPart, error) {
|
|
partType := strings.TrimSpace(kitutil.Interface2String(part["type"]))
|
|
switch partType {
|
|
case "input_text", "output_text", "text":
|
|
text := kitutil.Interface2String(part["text"])
|
|
if text == "" {
|
|
return nil, nil
|
|
}
|
|
return []dto.GeminiPart{{Text: text}}, nil
|
|
case "input_image", "input_file", "input_audio", "input_video":
|
|
source := ContentPartToFileSource(part)
|
|
if source == nil {
|
|
return nil, nil
|
|
}
|
|
base64Data, mimeType, err := relaymedia.ResolveBase64Data(c, source, "formatting Responses input for Gemini")
|
|
if err != nil {
|
|
return nil, fmt.Errorf("get file data from '%s' failed: %w", source.GetIdentifier(), err)
|
|
}
|
|
if !sharedgemini.IsSupportedMimeType(mimeType) {
|
|
return nil, fmt.Errorf("mime type is not supported by Gemini: '%s', url: '%s', supported types are: %v", mimeType, source.GetIdentifier(), sharedgemini.SupportedMimeTypesList())
|
|
}
|
|
return []dto.GeminiPart{
|
|
{
|
|
InlineData: &dto.GeminiInlineData{
|
|
MimeType: mimeType,
|
|
Data: base64Data,
|
|
},
|
|
},
|
|
}, nil
|
|
default:
|
|
return nil, nil
|
|
}
|
|
}
|
|
|
|
func responsesFunctionCallItemToGeminiPart(item map[string]any, itemType string) (dto.GeminiPart, string, error) {
|
|
name := responsesCallName(item)
|
|
if name == "" {
|
|
return dto.GeminiPart{}, "", fmt.Errorf("%s item is missing name", itemType)
|
|
}
|
|
var arguments map[string]any
|
|
if itemType == ResponsesInputTypeCustomToolCall {
|
|
// The custom tool is declared as a function taking one string
|
|
// argument, so its raw input is replayed in that shape.
|
|
arguments = map[string]any{convmeta.CustomToolInputArgument: responsesArgumentsString(item["input"])}
|
|
} else {
|
|
arguments = ObjectValue(item["arguments"], "arguments")
|
|
}
|
|
callID := CallID(item)
|
|
return dto.GeminiPart{
|
|
FunctionCall: &dto.FunctionCall{
|
|
ID: callID,
|
|
FunctionName: name,
|
|
Arguments: arguments,
|
|
},
|
|
}, callID, nil
|
|
}
|
|
|
|
// responsesFunctionOutputItemToGeminiPart returns the functionResponse part
|
|
// and, unless mediaInParts puts them into functionResponse.parts, the tool
|
|
// media the caller sends as separate parts after the turn's function
|
|
// responses.
|
|
func responsesFunctionOutputItemToGeminiPart(c context.Context, item map[string]any, callNames map[string]string, mediaInParts bool) (dto.GeminiPart, []dto.GeminiPart, error) {
|
|
callID := CallID(item)
|
|
name := strings.TrimSpace(kitutil.Interface2String(item["name"]))
|
|
if name == "" {
|
|
name = callNames[callID]
|
|
}
|
|
output, media := responsesToolOutputToGemini(c, item["output"], mediaInParts)
|
|
response := &dto.GeminiFunctionResponse{
|
|
Name: name,
|
|
Response: output,
|
|
}
|
|
if mediaInParts && len(media) > 0 {
|
|
parts, err := kitutil.Marshal(media)
|
|
if err != nil {
|
|
return dto.GeminiPart{}, nil, fmt.Errorf("failed to marshal function response parts: %w", err)
|
|
}
|
|
response.Parts = parts
|
|
media = nil
|
|
}
|
|
if callID != "" {
|
|
id, err := kitutil.Marshal(callID)
|
|
if err != nil {
|
|
return dto.GeminiPart{}, nil, fmt.Errorf("failed to marshal function response ID: %w", err)
|
|
}
|
|
response.ID = id
|
|
}
|
|
return dto.GeminiPart{
|
|
FunctionResponse: response,
|
|
}, media, nil
|
|
}
|
|
|
|
// responsesToolOutputToGemini maps a function_call_output payload onto a Gemini
|
|
// functionResponse. A Responses content-part array keeps its text in response
|
|
// and returns its media as inlineData parts; stringifying them instead would
|
|
// hand base64 media to the upstream text tokenizer. Multimodal function
|
|
// responses (mediaInParts) carry images and documents only; separate parts
|
|
// carry every type Gemini accepts. Media that cannot be loaded or carried is
|
|
// omitted with a diagnostic. An array holding any other part type, and every
|
|
// other payload shape, keeps the historical response map.
|
|
func responsesToolOutputToGemini(c context.Context, value any, mediaInParts bool) (map[string]any, []dto.GeminiPart) {
|
|
rawParts, ok := value.([]any)
|
|
if !ok || len(rawParts) == 0 {
|
|
return GeminiResponseMap(value), nil
|
|
}
|
|
parts := make([]map[string]any, 0, len(rawParts))
|
|
for _, rawPart := range rawParts {
|
|
part, isMap := rawPart.(map[string]any)
|
|
if !isMap {
|
|
return GeminiResponseMap(value), nil
|
|
}
|
|
switch strings.TrimSpace(kitutil.Interface2String(part["type"])) {
|
|
case "input_text", "output_text", "text", "input_image", "input_file", "input_audio", "input_video":
|
|
default:
|
|
return GeminiResponseMap(value), nil
|
|
}
|
|
parts = append(parts, part)
|
|
}
|
|
|
|
texts := make([]string, 0, len(parts))
|
|
labels := make([]string, 0, len(parts))
|
|
media := make([]dto.GeminiPart, 0, len(parts))
|
|
for _, part := range parts {
|
|
partType := strings.TrimSpace(kitutil.Interface2String(part["type"]))
|
|
switch partType {
|
|
case "input_text", "output_text", "text":
|
|
if text := kitutil.Interface2String(part["text"]); text != "" {
|
|
texts = append(texts, text)
|
|
}
|
|
continue
|
|
}
|
|
kind := strings.TrimPrefix(partType, "input_")
|
|
if label := "[" + kind + "]"; !slices.Contains(labels, label) {
|
|
labels = append(labels, label)
|
|
}
|
|
source := ContentPartToFileSource(part)
|
|
if source == nil {
|
|
continue
|
|
}
|
|
// The documented MIME list for multimodal function responses has no
|
|
// audio or video, and Gemini 3 ignores audio sent there.
|
|
reason := fmt.Sprintf("Gemini function responses cannot carry %s content", kind)
|
|
if !mediaInParts || kind == "image" || kind == "file" {
|
|
base64Data, mimeType, err := relaymedia.ResolveBase64Data(c, source, "formatting Responses tool output for Gemini")
|
|
topLevelType, _, _ := strings.Cut(strings.ToLower(mimeType), "/")
|
|
switch {
|
|
case err != nil:
|
|
reason = fmt.Sprintf("the %s could not be loaded", kind)
|
|
case sharedgemini.IsSupportedMimeType(mimeType) && (!mediaInParts || topLevelType != "audio" && topLevelType != "video"):
|
|
media = append(media, dto.GeminiPart{InlineData: &dto.GeminiInlineData{MimeType: mimeType, Data: base64Data}})
|
|
continue
|
|
default:
|
|
reason = fmt.Sprintf("Gemini function responses cannot carry %s content", mimeType)
|
|
}
|
|
}
|
|
convdiag.Add(c, types.ConversionDiagnostic{
|
|
Code: "unsupported_media_type",
|
|
Path: "input.output",
|
|
Message: reason + "; the part was omitted",
|
|
Severity: types.ConversionDiagnosticError,
|
|
})
|
|
}
|
|
|
|
switch {
|
|
case len(texts) > 0:
|
|
return GeminiResponseMap(strings.Join(texts, "\n")), media
|
|
case len(media) > 0:
|
|
return map[string]any{}, media
|
|
default:
|
|
// Every media part was omitted; name what was there instead of
|
|
// sending the payload as text.
|
|
return GeminiResponseMap(strings.Join(labels, " ")), nil
|
|
}
|
|
}
|
|
|
|
func appendGeminiContentPart(req *dto.GeminiChatRequest, role string, part dto.GeminiPart) {
|
|
if len(req.Contents) > 0 && req.Contents[len(req.Contents)-1].Role == role {
|
|
parts := req.Contents[len(req.Contents)-1].Parts
|
|
insertAt := len(parts)
|
|
switch {
|
|
case role == "model" && part.FunctionCall != nil:
|
|
insertAt = 0
|
|
for insertAt < len(parts) && parts[insertAt].FunctionCall != nil {
|
|
insertAt++
|
|
}
|
|
case part.FunctionResponse != nil:
|
|
// Tool media sent as separate parts stay after every function
|
|
// response of the turn, so a later response goes before them.
|
|
for i := len(parts) - 1; i >= 0; i-- {
|
|
if parts[i].FunctionResponse != nil {
|
|
insertAt = i + 1
|
|
break
|
|
}
|
|
}
|
|
}
|
|
req.Contents[len(req.Contents)-1].Parts = slices.Insert(parts, insertAt, part)
|
|
return
|
|
}
|
|
req.Contents = append(req.Contents, dto.GeminiChatContent{
|
|
Role: role,
|
|
Parts: []dto.GeminiPart{part},
|
|
})
|
|
}
|
|
|
|
func responsesGeminiRole(item map[string]any) string {
|
|
switch strings.TrimSpace(kitutil.Interface2String(item["role"])) {
|
|
case "assistant":
|
|
return "model"
|
|
case "system", "developer":
|
|
return "system"
|
|
case "model":
|
|
return "model"
|
|
default:
|
|
return "user"
|
|
}
|
|
}
|