296 lines
9.3 KiB
Go
296 lines
9.3 KiB
Go
package gemini
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"maps"
|
|
"slices"
|
|
"strconv"
|
|
"strings"
|
|
|
|
"github.com/QuantumNous/new-api/relaykit/dto"
|
|
"github.com/QuantumNous/new-api/relaykit/relayconvert/convmeta"
|
|
"github.com/QuantumNous/new-api/relaykit/relayconvert/internal/convdiag"
|
|
"github.com/QuantumNous/new-api/relaykit/relayconvert/reasoning"
|
|
"github.com/QuantumNous/new-api/relaykit/types"
|
|
)
|
|
|
|
// supportedMimeTypes follows the Blob.mimeType list in the Gemini API
|
|
// reference (https://ai.google.dev/api/generate-content#Blob). Audio and
|
|
// video are listed there as audio/* and video/*; IsSupportedMimeType accepts
|
|
// them by prefix.
|
|
var supportedMimeTypes = map[string]bool{
|
|
"image/png": true,
|
|
"image/jpeg": true,
|
|
"image/jpg": true,
|
|
"image/webp": true,
|
|
"image/heic": true,
|
|
"image/heif": true,
|
|
"image/gif": true,
|
|
"image/avif": true,
|
|
"text/plain": true,
|
|
"text/html": true,
|
|
"text/css": true,
|
|
"text/javascript": true,
|
|
"text/x-typescript": true,
|
|
"text/csv": true,
|
|
"text/markdown": true,
|
|
"text/x-python": true,
|
|
"text/xml": true,
|
|
"text/rtf": true,
|
|
"application/x-javascript": true,
|
|
"application/x-typescript": true,
|
|
"application/x-python-code": true,
|
|
"application/json": true,
|
|
"application/x-ipynb+json": true,
|
|
"application/rtf": true,
|
|
"application/pdf": true,
|
|
}
|
|
|
|
// IsSupportedMimeType reports whether Gemini accepts inline data of the MIME
|
|
// type.
|
|
func IsSupportedMimeType(mimeType string) bool {
|
|
mimeType = strings.ToLower(mimeType)
|
|
return supportedMimeTypes[mimeType] || strings.HasPrefix(mimeType, "audio/") || strings.HasPrefix(mimeType, "video/")
|
|
}
|
|
|
|
var SafetySettingCategories = []string{
|
|
"HARM_CATEGORY_HARASSMENT",
|
|
"HARM_CATEGORY_HATE_SPEECH",
|
|
"HARM_CATEGORY_SEXUALLY_EXPLICIT",
|
|
"HARM_CATEGORY_DANGEROUS_CONTENT",
|
|
}
|
|
|
|
const ThoughtSignatureBypassValue = "context_engineering_is_the_way_to_go"
|
|
|
|
func ShouldAttachThoughtSignature(opts *convmeta.Options) bool {
|
|
return opts != nil && opts.Gemini.FunctionCallThoughtSignatureEnabled
|
|
}
|
|
|
|
func AttachThoughtSignatureBypass(opts *convmeta.Options, part *dto.GeminiPart) bool {
|
|
if part == nil || len(part.ThoughtSignature) > 0 || !ShouldAttachThoughtSignature(opts) {
|
|
return false
|
|
}
|
|
part.ThoughtSignature = []byte(strconv.Quote(ThoughtSignatureBypassValue))
|
|
return true
|
|
}
|
|
|
|
func AttachFunctionCallThoughtSignature(opts *convmeta.Options, part *dto.GeminiPart) bool {
|
|
if part == nil || !HasFunctionCallContent(part.FunctionCall) {
|
|
return false
|
|
}
|
|
return AttachThoughtSignatureBypass(opts, part)
|
|
}
|
|
|
|
func AttachFirstTextThoughtSignature(opts *convmeta.Options, parts []dto.GeminiPart) bool {
|
|
if !ShouldAttachThoughtSignature(opts) {
|
|
return false
|
|
}
|
|
for i := range parts {
|
|
if parts[i].Text != "" && len(parts[i].ThoughtSignature) == 0 {
|
|
parts[i].ThoughtSignature = []byte(strconv.Quote(ThoughtSignatureBypassValue))
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// ApplyThinkingConfig resolves every reasoning control that can reach a
|
|
// Gemini request (provider-native thinkingConfig, generic OpenAI fields, and
|
|
// host model-name aliases) into the target model's dialect. Precedence is
|
|
// model name, then provider-native config, then generic fields; each override
|
|
// and each capability coercion is reported through convdiag on ctx.
|
|
func ApplyThinkingConfig(ctx context.Context, geminiRequest *dto.GeminiChatRequest, info convmeta.Meta, oaiRequest ...dto.GeneralOpenAIRequest) error {
|
|
opts := convmeta.OptionsOf(info)
|
|
if geminiRequest == nil {
|
|
return nil
|
|
}
|
|
|
|
modelName := convmeta.UpstreamModelName(info)
|
|
var source reasoning.Intent
|
|
crossProtocol := len(oaiRequest) > 0
|
|
if len(oaiRequest) > 0 {
|
|
if modelName == "" {
|
|
modelName = oaiRequest[0].Model
|
|
}
|
|
var diagnostics []types.ConversionDiagnostic
|
|
var err error
|
|
source, diagnostics, err = reasoning.FromOpenAIChat(&oaiRequest[0])
|
|
if err != nil {
|
|
return err
|
|
}
|
|
convdiag.Add(ctx, diagnostics...)
|
|
}
|
|
|
|
baseModel := modelName
|
|
suffix := reasoning.IntentFromState(convmeta.ReasoningStateOf(info))
|
|
preserveSuffix := opts.ShouldPreserveThinkingSuffix(modelName)
|
|
if info != nil && opts.ShouldPreserveThinkingSuffix(info.GetOriginModelName()) {
|
|
preserveSuffix = true
|
|
}
|
|
if preserveSuffix {
|
|
suffix = reasoning.Intent{}
|
|
}
|
|
// Native Gemini requests already use the target protocol. Without a host
|
|
// modifier, read portable effort metadata without running the capability
|
|
// renderer or rewriting provider-native controls.
|
|
if !crossProtocol && suffix.IsEmpty() {
|
|
if info != nil {
|
|
effort := ""
|
|
if config := geminiRequest.GenerationConfig.ThinkingConfig; config != nil {
|
|
effort = config.ThinkingLevel
|
|
// Gemini accepts thinkingLevel case-insensitively; record the
|
|
// canonical effort so logs match other protocols. Unknown values
|
|
// stay as sent because this path does not validate.
|
|
if canonical, err := reasoning.ParseEffort(effort); err == nil {
|
|
effort = string(canonical)
|
|
}
|
|
if effort == "" && config.ThinkingBudget != nil {
|
|
effort = string(reasoning.EffortFromBudget(*config.ThinkingBudget))
|
|
}
|
|
}
|
|
info.SetReasoningEffort(effort)
|
|
}
|
|
return nil
|
|
}
|
|
// Rewrite the provider-native config into the target model's dialect
|
|
// first, so a budget on Gemini 3 or a level on Gemini 2.5 is converted
|
|
// rather than rejected and later comparisons see what will be sent.
|
|
nativeEffort, diagnostics, err := reasoning.NormalizeGeminiThinkingConfig(baseModel, &geminiRequest.GenerationConfig)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
convdiag.Add(ctx, diagnostics...)
|
|
native, diagnostics, err := reasoning.FromGemini(geminiRequest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
convdiag.Add(ctx, diagnostics...)
|
|
if native.HasStrength() && source.HasStrength() {
|
|
// Native Gemini configuration is the lossless representation and wins
|
|
// over the generic OpenAI fields. Only a generic effort or budget can
|
|
// disagree with it; a bare enable states no strength to compare.
|
|
if source.Effort != "" || source.BudgetTokens != nil {
|
|
equivalent, compareErr := reasoning.EquivalentGeminiStrength(baseModel, native, source)
|
|
if compareErr != nil {
|
|
return compareErr
|
|
}
|
|
if !equivalent {
|
|
convdiag.Add(ctx, types.ConversionDiagnostic{
|
|
Code: "native_overrode_standard",
|
|
Path: "generationConfig.thinkingConfig",
|
|
Message: fmt.Sprintf("model %q: Gemini thinking_config effort %q overrides the standard reasoning effort %q", modelName, reasoning.EffectiveEffort(native), reasoning.EffectiveEffort(source)),
|
|
Severity: types.ConversionDiagnosticWarning,
|
|
})
|
|
}
|
|
}
|
|
if native.IncludeThoughts == nil {
|
|
native.IncludeThoughts = source.IncludeThoughts
|
|
}
|
|
source = reasoning.Intent{}
|
|
}
|
|
explicit, diagnostics, err := reasoning.MergeExplicit(native, source, modelName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
convdiag.Add(ctx, diagnostics...)
|
|
if explicit.HasStrength() && suffix.HasStrength() {
|
|
equivalent, compareErr := reasoning.EquivalentGeminiStrength(baseModel, explicit, suffix)
|
|
if compareErr != nil {
|
|
return compareErr
|
|
}
|
|
if equivalent {
|
|
if explicit.IncludeThoughts == nil {
|
|
explicit.IncludeThoughts = suffix.IncludeThoughts
|
|
}
|
|
suffix = reasoning.Intent{}
|
|
}
|
|
}
|
|
requested, diagnostics, err := reasoning.MergeExplicitAndSuffix(explicit, suffix, modelName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
convdiag.Add(ctx, diagnostics...)
|
|
requested = reasoning.ResolveGeminiEnabledDefault(baseModel, requested, geminiRequest.GenerationConfig.MaxOutputTokens)
|
|
|
|
if native.HasStrength() && !suffix.HasStrength() {
|
|
// The normalized native config is what goes upstream; only portable
|
|
// visibility metadata is taken from the standard representation.
|
|
if explicit.IncludeThoughts != nil {
|
|
geminiRequest.GenerationConfig.ThinkingConfig.IncludeThoughts = explicit.IncludeThoughts
|
|
}
|
|
if info != nil && nativeEffort != "" {
|
|
info.SetReasoningEffort(string(nativeEffort))
|
|
}
|
|
return nil
|
|
}
|
|
if requested.IsEmpty() {
|
|
return nil
|
|
}
|
|
rendered, err := reasoning.RenderGemini(
|
|
baseModel,
|
|
requested,
|
|
geminiRequest.GenerationConfig.MaxOutputTokens,
|
|
opts.Gemini.ThinkingAdapterBudgetTokensPercentage,
|
|
)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
convdiag.Add(ctx, rendered.Diagnostics...)
|
|
geminiRequest.GenerationConfig.ThinkingConfig = rendered.Config
|
|
if info != nil && rendered.EffectiveEffort != "" {
|
|
info.SetReasoningEffort(string(rendered.EffectiveEffort))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func ParseStopSequences(stop any) []string {
|
|
if stop == nil {
|
|
return nil
|
|
}
|
|
|
|
switch v := stop.(type) {
|
|
case string:
|
|
if v != "" {
|
|
return []string{v}
|
|
}
|
|
case []string:
|
|
return v
|
|
case []interface{}:
|
|
sequences := make([]string, 0, len(v))
|
|
for _, item := range v {
|
|
if str, ok := item.(string); ok && str != "" {
|
|
sequences = append(sequences, str)
|
|
}
|
|
}
|
|
return sequences
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func HasFunctionCallContent(call *dto.FunctionCall) bool {
|
|
if call == nil {
|
|
return false
|
|
}
|
|
if strings.TrimSpace(call.FunctionName) != "" {
|
|
return true
|
|
}
|
|
|
|
switch v := call.Arguments.(type) {
|
|
case nil:
|
|
return false
|
|
case string:
|
|
return strings.TrimSpace(v) != ""
|
|
case map[string]interface{}:
|
|
return len(v) > 0
|
|
case []interface{}:
|
|
return len(v) > 0
|
|
default:
|
|
return true
|
|
}
|
|
}
|
|
|
|
func SupportedMimeTypesList() []string {
|
|
return append(slices.Sorted(maps.Keys(supportedMimeTypes)), "audio/*", "video/*")
|
|
}
|