feat: publish sanitized custom.97 source

relaykit/relayconvert/internal/shared/claude/media.go

@@ -0,0 +1,63 @@
+package claude
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/QuantumNous/new-api/relaykit/dto"
+	relaymedia "github.com/QuantumNous/new-api/relaykit/relayconvert/internal/media"
+)
+
+// imageMediaTypes are the image formats Claude accepts.
+var imageMediaTypes = map[string]struct{}{
+	"image/jpeg": {},
+	"image/png":  {},
+	"image/gif":  {},
+	"image/webp": {},
+}
+
+// MediaBlock builds the Claude content block for resolved base64 media: an
+// image in a format Claude accepts, a PDF document, or a plain-text document
+// for text files. declaredImage marks media sent as an image part, which
+// passes through when its MIME type is unknown. A non-empty reason means
+// Claude cannot carry the media and the block must be omitted.
+func MediaBlock(base64Data string, mimeType string, declaredImage bool) (dto.ClaudeMediaMessage, string) {
+	mediaType, _, _ := strings.Cut(strings.ToLower(strings.TrimSpace(mimeType)), ";")
+	mediaType = strings.TrimSpace(mediaType)
+	if mediaType == "image/jpg" {
+		mediaType = "image/jpeg"
+	}
+	switch {
+	case mediaType == "application/pdf":
+		return dto.ClaudeMediaMessage{
+			Type:   "document",
+			Source: &dto.ClaudeMessageSource{Type: "base64", MediaType: mediaType, Data: base64Data},
+		}, ""
+	case relaymedia.IsTextMimeType(mediaType):
+		text, ok := relaymedia.DecodeText(base64Data)
+		if !ok {
+			return dto.ClaudeMediaMessage{}, fmt.Sprintf("text file of MIME type %q could not be decoded as UTF-8 text", mimeType)
+		}
+		return dto.ClaudeMediaMessage{
+			Type:   "document",
+			Source: &dto.ClaudeMessageSource{Type: "text", MediaType: "text/plain", Data: text},
+		}, ""
+	case mediaType == "" && declaredImage:
+		return dto.ClaudeMediaMessage{
+			Type:   "image",
+			Source: &dto.ClaudeMessageSource{Type: "base64", MediaType: mimeType, Data: base64Data},
+		}, ""
+	case mediaType == "application/octet-stream" && declaredImage:
+		return dto.ClaudeMediaMessage{}, "the image format could not be identified"
+	}
+	if _, ok := imageMediaTypes[mediaType]; ok {
+		return dto.ClaudeMediaMessage{
+			Type:   "image",
+			Source: &dto.ClaudeMessageSource{Type: "base64", MediaType: mediaType, Data: base64Data},
+		}, ""
+	}
+	if strings.HasPrefix(mediaType, "image/") {
+		return dto.ClaudeMediaMessage{}, fmt.Sprintf("Claude accepts only JPEG, PNG, GIF, and WebP images, not %q", mimeType)
+	}
+	return dto.ClaudeMediaMessage{}, fmt.Sprintf("Claude Messages cannot carry content of MIME type %q", mimeType)
+}

relaykit/relayconvert/internal/shared/claude/reasoning.go

@@ -44,7 +44,7 @@ func ApplyReasoning(ctx context.Context, req *dto.ClaudeRequest, info convmeta.M
 				case req.Thinking.BudgetTokens != nil:
 					effort = string(reasoning.EffortFromBudget(*req.Thinking.BudgetTokens))
 				case req.Thinking.Type == "enabled" || req.Thinking.Type == "adaptive":
-					effort = string(reasoning.EffortHigh)
+					effort = string(reasoning.ClaudeDefaultEffort(req.Model))
 				}
 			}
 			info.SetReasoningEffort(effort)
@@ -81,6 +81,16 @@ func ApplyReasoning(ctx context.Context, req *dto.ClaudeRequest, info convmeta.M
 			intent.Effort = reasoning.EffortHigh
 		}
 	}
+	requestEffort, err := reasoning.ParseEffort(req.GetEfforts())
+	if err != nil {
+		return err
+	}
+	if knownClaudeModel && intent.Mode == reasoning.ModeDisabled && (intent.Effort == "" || intent.Effort == reasoning.EffortNone) &&
+		requestEffort != "" && requestEffort != reasoning.EffortNone {
+		// Effort works without thinking, so the request's own effort is
+		// rendered for the thinking-off state rather than left unchecked.
+		intent.Effort = requestEffort
+	}
 	if req.MaxTokens == nil && intent.HasStrength() {
 		// Adapter-provided defaults may be raised to accommodate an exact
 		// cross-protocol budget. Values too small for manual thinking are
@@ -117,7 +127,14 @@ func ApplyReasoning(ctx context.Context, req *dto.ClaudeRequest, info convmeta.M
 	if rendered.Thinking != nil {
 		req.Thinking = rendered.Thinking
 	}
-	if rendered.OutputEffort != "" {
+	outputEffort := rendered.OutputEffort
+	// An effort the request still carries reaches a known model only at a
+	// level it accepts, and is removed where the model takes none.
+	rewriteRequestEffort := outputEffort == "" && knownClaudeModel && requestEffort != ""
+	if rewriteRequestEffort {
+		outputEffort = reasoning.ClaudeEffort(capabilityModel, requestEffort)
+	}
+	if outputEffort != "" || rewriteRequestEffort {
 		outputConfig := make(map[string]any)
 		if len(req.OutputConfig) > 0 {
 			if kitutil.GetJsonType(req.OutputConfig) != "object" {
@@ -130,12 +147,20 @@ func ApplyReasoning(ctx context.Context, req *dto.ClaudeRequest, info convmeta.M
 				outputConfig = make(map[string]any)
 			}
 		}
-		outputConfig["effort"] = string(rendered.OutputEffort)
-		encoded, err := kitutil.Marshal(outputConfig)
-		if err != nil {
-			return fmt.Errorf("failed to marshal Claude output_config: %w", err)
+		if outputEffort != "" {
+			outputConfig["effort"] = string(outputEffort)
+		} else {
+			delete(outputConfig, "effort")
+		}
+		if len(outputConfig) == 0 {
+			req.OutputConfig = nil
+		} else {
+			encoded, err := kitutil.Marshal(outputConfig)
+			if err != nil {
+				return fmt.Errorf("failed to marshal Claude output_config: %w", err)
+			}
+			req.OutputConfig = encoded
 		}
-		req.OutputConfig = encoded
 	}
 	if rendered.ClearSampling {
 		if req.Temperature != nil || req.TopP != nil || req.TopK != nil {

relaykit/relayconvert/internal/shared/gemini/request.go

@@ -3,6 +3,8 @@ package gemini
 import (
 	"context"
 	"fmt"
+	"maps"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -13,26 +15,43 @@ import (
 	"github.com/QuantumNous/new-api/relaykit/types"
 )
 
-var SupportedMimeTypes = map[string]bool{
-	"application/pdf": true,
-	"audio/mpeg":      true,
-	"audio/mp3":       true,
-	"audio/wav":       true,
-	"image/png":       true,
-	"image/jpeg":      true,
-	"image/jpg":       true,
-	"image/webp":      true,
-	"image/heic":      true,
-	"image/heif":      true,
-	"text/plain":      true,
-	"video/mov":       true,
-	"video/mpeg":      true,
-	"video/mp4":       true,
-	"video/mpg":       true,
-	"video/avi":       true,
-	"video/wmv":       true,
-	"video/mpegps":    true,
-	"video/flv":       true,
+// supportedMimeTypes follows the Blob.mimeType list in the Gemini API
+// reference (https://ai.google.dev/api/generate-content#Blob). Audio and
+// video are listed there as audio/* and video/*; IsSupportedMimeType accepts
+// them by prefix.
+var supportedMimeTypes = map[string]bool{
+	"image/png":                 true,
+	"image/jpeg":                true,
+	"image/jpg":                 true,
+	"image/webp":                true,
+	"image/heic":                true,
+	"image/heif":                true,
+	"image/gif":                 true,
+	"image/avif":                true,
+	"text/plain":                true,
+	"text/html":                 true,
+	"text/css":                  true,
+	"text/javascript":           true,
+	"text/x-typescript":         true,
+	"text/csv":                  true,
+	"text/markdown":             true,
+	"text/x-python":             true,
+	"text/xml":                  true,
+	"text/rtf":                  true,
+	"application/x-javascript":  true,
+	"application/x-typescript":  true,
+	"application/x-python-code": true,
+	"application/json":          true,
+	"application/x-ipynb+json":  true,
+	"application/rtf":           true,
+	"application/pdf":           true,
+}
+
+// IsSupportedMimeType reports whether Gemini accepts inline data of the MIME
+// type.
+func IsSupportedMimeType(mimeType string) bool {
+	mimeType = strings.ToLower(mimeType)
+	return supportedMimeTypes[mimeType] || strings.HasPrefix(mimeType, "audio/") || strings.HasPrefix(mimeType, "video/")
 }
 
 var SafetySettingCategories = []string{
@@ -272,9 +291,5 @@ func HasFunctionCallContent(call *dto.FunctionCall) bool {
 }
 
 func SupportedMimeTypesList() []string {
-	keys := make([]string, 0, len(SupportedMimeTypes))
-	for key := range SupportedMimeTypes {
-		keys = append(keys, key)
-	}
-	return keys
+	return append(slices.Sorted(maps.Keys(supportedMimeTypes)), "audio/*", "video/*")
 }

relaykit/relayconvert/internal/shared/gemini/schema.go

@@ -1,9 +1,17 @@
 package gemini
 
 import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strconv"
 	"strings"
 
 	"github.com/QuantumNous/new-api/relaykit/dto"
+	"github.com/QuantumNous/new-api/relaykit/relayconvert/internal/convdiag"
+	kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil"
+	"github.com/QuantumNous/new-api/relaykit/types"
 )
 
 var geminiOpenAPISchemaAllowedFields = map[string]struct{}{
@@ -56,6 +64,7 @@ func cleanGeminiFunctionParametersWithDepth(params interface{}, depth int) inter
 		}
 
 		normalizeGeminiSchemaTypeAndNullable(cleanedMap)
+		sanitizeGeminiEnum(cleanedMap)
 
 		if props, ok := cleanedMap["properties"].(map[string]interface{}); ok && props != nil {
 			cleanedProps := make(map[string]interface{})
@@ -102,6 +111,7 @@ func cleanGeminiFunctionParametersShallow(params interface{}) interface{} {
 			}
 		}
 		normalizeGeminiSchemaTypeAndNullable(cleanedMap)
+		sanitizeGeminiEnum(cleanedMap)
 		delete(cleanedMap, "properties")
 		delete(cleanedMap, "items")
 		delete(cleanedMap, "anyOf")
@@ -113,6 +123,30 @@ func cleanGeminiFunctionParametersShallow(params interface{}) interface{} {
 	}
 }
 
+// sanitizeGeminiEnum keeps the enum constraint only when every element is a
+// string, the only form Gemini's legacy Schema accepts. Any other shape is
+// dropped (e.g. boolean or number elements, which valid JSON Schema allows but
+// Gemini rejects with a 400). This widens the value domain for that schema, a
+// deliberate lossy relaxation: the declared type is preserved, string enums
+// pass through untouched.
+func sanitizeGeminiEnum(schema map[string]interface{}) {
+	raw, exists := schema["enum"]
+	if !exists {
+		return
+	}
+	values, ok := raw.([]interface{})
+	if !ok {
+		delete(schema, "enum")
+		return
+	}
+	for _, value := range values {
+		if _, isString := value.(string); !isString {
+			delete(schema, "enum")
+			return
+		}
+	}
+}
+
 func normalizeGeminiSchemaTypeAndNullable(schema map[string]interface{}) {
 	rawType, ok := schema["type"]
 	if !ok || rawType == nil {
@@ -175,6 +209,148 @@ func normalizeGeminiSchemaTypeAndNullable(schema map[string]interface{}) {
 	}
 }
 
+// FunctionParameters returns the Gemini function declaration field that
+// carries schema, and its value. A schema that fits Gemini's typed Schema is
+// cleaned as before, and still left out when its properties are empty. Any
+// other schema goes unchanged to parametersJsonSchema when jsonSchema is
+// true; otherwise it is cleaned anyway and misfits names what the typed
+// Schema cannot carry.
+func FunctionParameters(schema any, jsonSchema bool) (field string, value any, misfits []string) {
+	misfits = schemaMisfits(schema)
+	object, _ := schema.(map[string]any)
+	properties, hasProperties := object["properties"].(map[string]any)
+	emptyObject := hasProperties && len(properties) == 0
+	if emptyObject {
+		for key, item := range object {
+			switch key {
+			case "type", "properties":
+				continue
+			case "required":
+				if required, _ := item.([]any); len(required) == 0 {
+					continue
+				}
+			}
+			if _, allowed := geminiOpenAPISchemaAllowedFields[key]; allowed {
+				misfits = append(misfits, "constraints on an empty object")
+				break
+			}
+		}
+	}
+	if len(misfits) > 0 && jsonSchema {
+		return "parametersJsonSchema", schema, nil
+	}
+	if emptyObject {
+		return "parameters", nil, misfits
+	}
+	return "parameters", CleanFunctionParameters(schema), misfits
+}
+
+// ApplyResponseSchema sets the structured output schema from the json_schema
+// object of an OpenAI response format. A schema that fits Gemini's typed
+// Schema keeps the responseSchema cleanup. Any other schema is sent unchanged
+// as responseJsonSchema when jsonSchema is true; otherwise it is cleaned
+// anyway and a diagnostic at path names what responseSchema cannot carry.
+func ApplyResponseSchema(ctx context.Context, config *dto.GeminiChatGenerationConfig, rawFormat json.RawMessage, path string, jsonSchema bool) {
+	var format dto.FormatJsonSchema
+	if err := kitutil.Unmarshal(rawFormat, &format); err != nil {
+		return
+	}
+	misfits := schemaMisfits(format.Schema)
+	if len(misfits) > 0 && jsonSchema {
+		// The raw member keeps the client's key order.
+		var raw struct {
+			Schema json.RawMessage `json:"schema"`
+		}
+		if err := kitutil.Unmarshal(rawFormat, &raw); err == nil {
+			config.ResponseJsonSchema = raw.Schema
+			return
+		}
+	}
+	config.ResponseSchema = RemoveAdditionalProperties(format.Schema, 0)
+	if len(misfits) > 0 {
+		convdiag.Add(ctx, types.ConversionDiagnostic{
+			Code:     "gemini_schema_keyword_dropped",
+			Path:     path,
+			Message:  fmt.Sprintf("Gemini response schema cannot carry %s", strings.Join(misfits, ", ")),
+			Severity: types.ConversionDiagnosticWarning,
+		})
+	}
+}
+
+// schemaMisfits lists what Gemini's typed Schema cannot hold of a JSON Schema
+// as written: keywords outside its fields, type arrays, non-string enum
+// values, tuple items, boolean schemas, and nesting deeper than
+// CleanFunctionParameters keeps. additionalProperties and $schema are not
+// misfits: agent tools and OpenAI strict schemas carry them on nearly every
+// object, and dropping them as before keeps those requests unchanged for
+// upstreams that do not pass the JSON Schema fields through.
+func schemaMisfits(schema any) []string {
+	found := make(map[string]struct{})
+	collectSchemaMisfits(schema, 0, found)
+	misfits := make([]string, 0, len(found))
+	for misfit := range found {
+		misfits = append(misfits, misfit)
+	}
+	slices.Sort(misfits)
+	return misfits
+}
+
+func collectSchemaMisfits(schema any, depth int, found map[string]struct{}) {
+	if depth > geminiFunctionSchemaMaxDepth {
+		found[fmt.Sprintf("nesting deeper than %d levels", geminiFunctionSchemaMaxDepth)] = struct{}{}
+		return
+	}
+	value, ok := schema.(map[string]any)
+	if !ok {
+		if _, isBool := schema.(bool); isBool {
+			found["boolean schemas"] = struct{}{}
+		}
+		return
+	}
+	for key, item := range value {
+		if _, allowed := geminiOpenAPISchemaAllowedFields[key]; !allowed {
+			if key != "additionalProperties" && key != "$schema" {
+				found[key] = struct{}{}
+			}
+			continue
+		}
+		switch key {
+		case "type":
+			if _, isArray := item.([]any); isArray {
+				found["type arrays"] = struct{}{}
+			}
+		case "enum":
+			values, _ := item.([]any)
+			if slices.ContainsFunc(values, func(value any) bool {
+				_, isString := value.(string)
+				return !isString
+			}) {
+				found["non-string enum values"] = struct{}{}
+			}
+		case "properties":
+			properties, _ := item.(map[string]any)
+			for _, property := range properties {
+				collectSchemaMisfits(property, depth+1, found)
+			}
+		case "items":
+			entries, isTuple := item.([]any)
+			if !isTuple {
+				collectSchemaMisfits(item, depth+1, found)
+				continue
+			}
+			found["tuple items"] = struct{}{}
+			for _, entry := range entries {
+				collectSchemaMisfits(entry, depth+1, found)
+			}
+		case "anyOf":
+			variants, _ := item.([]any)
+			for _, variant := range variants {
+				collectSchemaMisfits(variant, depth+1, found)
+			}
+		}
+	}
+}
+
 func RemoveAdditionalProperties(schema interface{}, depth int) interface{} {
 	if depth >= 5 {
 		return schema
@@ -254,3 +430,112 @@ func OpenAIToolChoiceToConfig(toolChoice any) *dto.ToolConfig {
 
 	return nil
 }
+
+// geminiSchemaInt64Fields are Gemini Schema bounds typed int64, which the
+// proto JSON encoding may send as strings.
+var geminiSchemaInt64Fields = map[string]struct{}{
+	"maxItems":      {},
+	"minItems":      {},
+	"maxLength":     {},
+	"minLength":     {},
+	"maxProperties": {},
+	"minProperties": {},
+}
+
+// OpenAPISchemaToJSONSchema rewrites a schema in Gemini's OpenAPI subset as
+// JSON Schema: type names are lowercased, nullable becomes a "null" type,
+// example becomes examples, and int64 bounds sent as strings become numbers.
+// It also returns the Gemini-only keywords JSON Schema cannot express, which
+// are dropped (propertyOrdering).
+func OpenAPISchemaToJSONSchema(schema any) (any, []string) {
+	dropped := make(map[string]struct{})
+	converted := openAPISchemaToJSONSchema(schema, 0, dropped)
+	keywords := make([]string, 0, len(dropped))
+	for keyword := range dropped {
+		keywords = append(keywords, keyword)
+	}
+	slices.Sort(keywords)
+	return converted, keywords
+}
+
+func openAPISchemaToJSONSchema(schema any, depth int, dropped map[string]struct{}) any {
+	value, ok := schema.(map[string]any)
+	if !ok || depth >= geminiFunctionSchemaMaxDepth {
+		return schema
+	}
+	converted := make(map[string]any, len(value))
+	nullable := false
+	for key, item := range value {
+		switch key {
+		case "type":
+			typeName, isString := item.(string)
+			if !isString {
+				converted[key] = item
+				continue
+			}
+			typeName = strings.ToLower(strings.TrimSpace(typeName))
+			if typeName != "" && typeName != "type_unspecified" {
+				converted[key] = typeName
+			}
+		case "nullable":
+			nullable, _ = item.(bool)
+		case "propertyOrdering":
+			dropped[key] = struct{}{}
+		case "example":
+			if _, exists := value["examples"]; exists {
+				dropped[key] = struct{}{}
+				continue
+			}
+			converted["examples"] = []any{item}
+		case "properties":
+			properties, isMap := item.(map[string]any)
+			if !isMap {
+				converted[key] = item
+				continue
+			}
+			convertedProperties := make(map[string]any, len(properties))
+			for name, property := range properties {
+				convertedProperties[name] = openAPISchemaToJSONSchema(property, depth+1, dropped)
+			}
+			converted[key] = convertedProperties
+		case "items":
+			converted[key] = openAPISchemaToJSONSchema(item, depth+1, dropped)
+		case "anyOf":
+			variants, isArray := item.([]any)
+			if !isArray {
+				converted[key] = item
+				continue
+			}
+			convertedVariants := make([]any, len(variants))
+			for index, variant := range variants {
+				convertedVariants[index] = openAPISchemaToJSONSchema(variant, depth+1, dropped)
+			}
+			converted[key] = convertedVariants
+		default:
+			if _, isInt64 := geminiSchemaInt64Fields[key]; isInt64 {
+				if text, isString := item.(string); isString {
+					if number, err := strconv.ParseInt(strings.TrimSpace(text), 10, 64); err == nil {
+						converted[key] = number
+						continue
+					}
+				}
+			}
+			converted[key] = item
+		}
+	}
+	if !nullable {
+		return converted
+	}
+	switch typed := converted["type"].(type) {
+	case string:
+		converted["type"] = []any{typed, "null"}
+	default:
+		if variants, isArray := converted["anyOf"].([]any); isArray {
+			converted["anyOf"] = append(variants, map[string]any{"type": "null"})
+		}
+	}
+	if values, isArray := converted["enum"].([]any); isArray && !slices.Contains(values, nil) {
+		converted["enum"] = append(values, nil)
+	}
+	return converted
+}