feat: publish sanitized custom.97 source

relaykit/relayconvert/internal/shared/gemini/request.go

@@ -3,6 +3,8 @@ package gemini
 import (
 	"context"
 	"fmt"
+	"maps"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -13,26 +15,43 @@ import (
 	"github.com/QuantumNous/new-api/relaykit/types"
 )
 
-var SupportedMimeTypes = map[string]bool{
-	"application/pdf": true,
-	"audio/mpeg":      true,
-	"audio/mp3":       true,
-	"audio/wav":       true,
-	"image/png":       true,
-	"image/jpeg":      true,
-	"image/jpg":       true,
-	"image/webp":      true,
-	"image/heic":      true,
-	"image/heif":      true,
-	"text/plain":      true,
-	"video/mov":       true,
-	"video/mpeg":      true,
-	"video/mp4":       true,
-	"video/mpg":       true,
-	"video/avi":       true,
-	"video/wmv":       true,
-	"video/mpegps":    true,
-	"video/flv":       true,
+// supportedMimeTypes follows the Blob.mimeType list in the Gemini API
+// reference (https://ai.google.dev/api/generate-content#Blob). Audio and
+// video are listed there as audio/* and video/*; IsSupportedMimeType accepts
+// them by prefix.
+var supportedMimeTypes = map[string]bool{
+	"image/png":                 true,
+	"image/jpeg":                true,
+	"image/jpg":                 true,
+	"image/webp":                true,
+	"image/heic":                true,
+	"image/heif":                true,
+	"image/gif":                 true,
+	"image/avif":                true,
+	"text/plain":                true,
+	"text/html":                 true,
+	"text/css":                  true,
+	"text/javascript":           true,
+	"text/x-typescript":         true,
+	"text/csv":                  true,
+	"text/markdown":             true,
+	"text/x-python":             true,
+	"text/xml":                  true,
+	"text/rtf":                  true,
+	"application/x-javascript":  true,
+	"application/x-typescript":  true,
+	"application/x-python-code": true,
+	"application/json":          true,
+	"application/x-ipynb+json":  true,
+	"application/rtf":           true,
+	"application/pdf":           true,
+}
+
+// IsSupportedMimeType reports whether Gemini accepts inline data of the MIME
+// type.
+func IsSupportedMimeType(mimeType string) bool {
+	mimeType = strings.ToLower(mimeType)
+	return supportedMimeTypes[mimeType] || strings.HasPrefix(mimeType, "audio/") || strings.HasPrefix(mimeType, "video/")
 }
 
 var SafetySettingCategories = []string{
@@ -272,9 +291,5 @@ func HasFunctionCallContent(call *dto.FunctionCall) bool {
 }
 
 func SupportedMimeTypesList() []string {
-	keys := make([]string, 0, len(SupportedMimeTypes))
-	for key := range SupportedMimeTypes {
-		keys = append(keys, key)
-	}
-	return keys
+	return append(slices.Sorted(maps.Keys(supportedMimeTypes)), "audio/*", "video/*")
 }

relaykit/relayconvert/internal/shared/gemini/schema.go

@@ -1,9 +1,17 @@
 package gemini
 
 import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"slices"
+	"strconv"
 	"strings"
 
 	"github.com/QuantumNous/new-api/relaykit/dto"
+	"github.com/QuantumNous/new-api/relaykit/relayconvert/internal/convdiag"
+	kitutil "github.com/QuantumNous/new-api/relaykit/relayconvert/kitutil"
+	"github.com/QuantumNous/new-api/relaykit/types"
 )
 
 var geminiOpenAPISchemaAllowedFields = map[string]struct{}{
@@ -56,6 +64,7 @@ func cleanGeminiFunctionParametersWithDepth(params interface{}, depth int) inter
 		}
 
 		normalizeGeminiSchemaTypeAndNullable(cleanedMap)
+		sanitizeGeminiEnum(cleanedMap)
 
 		if props, ok := cleanedMap["properties"].(map[string]interface{}); ok && props != nil {
 			cleanedProps := make(map[string]interface{})
@@ -102,6 +111,7 @@ func cleanGeminiFunctionParametersShallow(params interface{}) interface{} {
 			}
 		}
 		normalizeGeminiSchemaTypeAndNullable(cleanedMap)
+		sanitizeGeminiEnum(cleanedMap)
 		delete(cleanedMap, "properties")
 		delete(cleanedMap, "items")
 		delete(cleanedMap, "anyOf")
@@ -113,6 +123,30 @@ func cleanGeminiFunctionParametersShallow(params interface{}) interface{} {
 	}
 }
 
+// sanitizeGeminiEnum keeps the enum constraint only when every element is a
+// string, the only form Gemini's legacy Schema accepts. Any other shape is
+// dropped (e.g. boolean or number elements, which valid JSON Schema allows but
+// Gemini rejects with a 400). This widens the value domain for that schema, a
+// deliberate lossy relaxation: the declared type is preserved, string enums
+// pass through untouched.
+func sanitizeGeminiEnum(schema map[string]interface{}) {
+	raw, exists := schema["enum"]
+	if !exists {
+		return
+	}
+	values, ok := raw.([]interface{})
+	if !ok {
+		delete(schema, "enum")
+		return
+	}
+	for _, value := range values {
+		if _, isString := value.(string); !isString {
+			delete(schema, "enum")
+			return
+		}
+	}
+}
+
 func normalizeGeminiSchemaTypeAndNullable(schema map[string]interface{}) {
 	rawType, ok := schema["type"]
 	if !ok || rawType == nil {
@@ -175,6 +209,148 @@ func normalizeGeminiSchemaTypeAndNullable(schema map[string]interface{}) {
 	}
 }
 
+// FunctionParameters returns the Gemini function declaration field that
+// carries schema, and its value. A schema that fits Gemini's typed Schema is
+// cleaned as before, and still left out when its properties are empty. Any
+// other schema goes unchanged to parametersJsonSchema when jsonSchema is
+// true; otherwise it is cleaned anyway and misfits names what the typed
+// Schema cannot carry.
+func FunctionParameters(schema any, jsonSchema bool) (field string, value any, misfits []string) {
+	misfits = schemaMisfits(schema)
+	object, _ := schema.(map[string]any)
+	properties, hasProperties := object["properties"].(map[string]any)
+	emptyObject := hasProperties && len(properties) == 0
+	if emptyObject {
+		for key, item := range object {
+			switch key {
+			case "type", "properties":
+				continue
+			case "required":
+				if required, _ := item.([]any); len(required) == 0 {
+					continue
+				}
+			}
+			if _, allowed := geminiOpenAPISchemaAllowedFields[key]; allowed {
+				misfits = append(misfits, "constraints on an empty object")
+				break
+			}
+		}
+	}
+	if len(misfits) > 0 && jsonSchema {
+		return "parametersJsonSchema", schema, nil
+	}
+	if emptyObject {
+		return "parameters", nil, misfits
+	}
+	return "parameters", CleanFunctionParameters(schema), misfits
+}
+
+// ApplyResponseSchema sets the structured output schema from the json_schema
+// object of an OpenAI response format. A schema that fits Gemini's typed
+// Schema keeps the responseSchema cleanup. Any other schema is sent unchanged
+// as responseJsonSchema when jsonSchema is true; otherwise it is cleaned
+// anyway and a diagnostic at path names what responseSchema cannot carry.
+func ApplyResponseSchema(ctx context.Context, config *dto.GeminiChatGenerationConfig, rawFormat json.RawMessage, path string, jsonSchema bool) {
+	var format dto.FormatJsonSchema
+	if err := kitutil.Unmarshal(rawFormat, &format); err != nil {
+		return
+	}
+	misfits := schemaMisfits(format.Schema)
+	if len(misfits) > 0 && jsonSchema {
+		// The raw member keeps the client's key order.
+		var raw struct {
+			Schema json.RawMessage `json:"schema"`
+		}
+		if err := kitutil.Unmarshal(rawFormat, &raw); err == nil {
+			config.ResponseJsonSchema = raw.Schema
+			return
+		}
+	}
+	config.ResponseSchema = RemoveAdditionalProperties(format.Schema, 0)
+	if len(misfits) > 0 {
+		convdiag.Add(ctx, types.ConversionDiagnostic{
+			Code:     "gemini_schema_keyword_dropped",
+			Path:     path,
+			Message:  fmt.Sprintf("Gemini response schema cannot carry %s", strings.Join(misfits, ", ")),
+			Severity: types.ConversionDiagnosticWarning,
+		})
+	}
+}
+
+// schemaMisfits lists what Gemini's typed Schema cannot hold of a JSON Schema
+// as written: keywords outside its fields, type arrays, non-string enum
+// values, tuple items, boolean schemas, and nesting deeper than
+// CleanFunctionParameters keeps. additionalProperties and $schema are not
+// misfits: agent tools and OpenAI strict schemas carry them on nearly every
+// object, and dropping them as before keeps those requests unchanged for
+// upstreams that do not pass the JSON Schema fields through.
+func schemaMisfits(schema any) []string {
+	found := make(map[string]struct{})
+	collectSchemaMisfits(schema, 0, found)
+	misfits := make([]string, 0, len(found))
+	for misfit := range found {
+		misfits = append(misfits, misfit)
+	}
+	slices.Sort(misfits)
+	return misfits
+}
+
+func collectSchemaMisfits(schema any, depth int, found map[string]struct{}) {
+	if depth > geminiFunctionSchemaMaxDepth {
+		found[fmt.Sprintf("nesting deeper than %d levels", geminiFunctionSchemaMaxDepth)] = struct{}{}
+		return
+	}
+	value, ok := schema.(map[string]any)
+	if !ok {
+		if _, isBool := schema.(bool); isBool {
+			found["boolean schemas"] = struct{}{}
+		}
+		return
+	}
+	for key, item := range value {
+		if _, allowed := geminiOpenAPISchemaAllowedFields[key]; !allowed {
+			if key != "additionalProperties" && key != "$schema" {
+				found[key] = struct{}{}
+			}
+			continue
+		}
+		switch key {
+		case "type":
+			if _, isArray := item.([]any); isArray {
+				found["type arrays"] = struct{}{}
+			}
+		case "enum":
+			values, _ := item.([]any)
+			if slices.ContainsFunc(values, func(value any) bool {
+				_, isString := value.(string)
+				return !isString
+			}) {
+				found["non-string enum values"] = struct{}{}
+			}
+		case "properties":
+			properties, _ := item.(map[string]any)
+			for _, property := range properties {
+				collectSchemaMisfits(property, depth+1, found)
+			}
+		case "items":
+			entries, isTuple := item.([]any)
+			if !isTuple {
+				collectSchemaMisfits(item, depth+1, found)
+				continue
+			}
+			found["tuple items"] = struct{}{}
+			for _, entry := range entries {
+				collectSchemaMisfits(entry, depth+1, found)
+			}
+		case "anyOf":
+			variants, _ := item.([]any)
+			for _, variant := range variants {
+				collectSchemaMisfits(variant, depth+1, found)
+			}
+		}
+	}
+}
+
 func RemoveAdditionalProperties(schema interface{}, depth int) interface{} {
 	if depth >= 5 {
 		return schema
@@ -254,3 +430,112 @@ func OpenAIToolChoiceToConfig(toolChoice any) *dto.ToolConfig {
 
 	return nil
 }
+
+// geminiSchemaInt64Fields are Gemini Schema bounds typed int64, which the
+// proto JSON encoding may send as strings.
+var geminiSchemaInt64Fields = map[string]struct{}{
+	"maxItems":      {},
+	"minItems":      {},
+	"maxLength":     {},
+	"minLength":     {},
+	"maxProperties": {},
+	"minProperties": {},
+}
+
+// OpenAPISchemaToJSONSchema rewrites a schema in Gemini's OpenAPI subset as
+// JSON Schema: type names are lowercased, nullable becomes a "null" type,
+// example becomes examples, and int64 bounds sent as strings become numbers.
+// It also returns the Gemini-only keywords JSON Schema cannot express, which
+// are dropped (propertyOrdering).
+func OpenAPISchemaToJSONSchema(schema any) (any, []string) {
+	dropped := make(map[string]struct{})
+	converted := openAPISchemaToJSONSchema(schema, 0, dropped)
+	keywords := make([]string, 0, len(dropped))
+	for keyword := range dropped {
+		keywords = append(keywords, keyword)
+	}
+	slices.Sort(keywords)
+	return converted, keywords
+}
+
+func openAPISchemaToJSONSchema(schema any, depth int, dropped map[string]struct{}) any {
+	value, ok := schema.(map[string]any)
+	if !ok || depth >= geminiFunctionSchemaMaxDepth {
+		return schema
+	}
+	converted := make(map[string]any, len(value))
+	nullable := false
+	for key, item := range value {
+		switch key {
+		case "type":
+			typeName, isString := item.(string)
+			if !isString {
+				converted[key] = item
+				continue
+			}
+			typeName = strings.ToLower(strings.TrimSpace(typeName))
+			if typeName != "" && typeName != "type_unspecified" {
+				converted[key] = typeName
+			}
+		case "nullable":
+			nullable, _ = item.(bool)
+		case "propertyOrdering":
+			dropped[key] = struct{}{}
+		case "example":
+			if _, exists := value["examples"]; exists {
+				dropped[key] = struct{}{}
+				continue
+			}
+			converted["examples"] = []any{item}
+		case "properties":
+			properties, isMap := item.(map[string]any)
+			if !isMap {
+				converted[key] = item
+				continue
+			}
+			convertedProperties := make(map[string]any, len(properties))
+			for name, property := range properties {
+				convertedProperties[name] = openAPISchemaToJSONSchema(property, depth+1, dropped)
+			}
+			converted[key] = convertedProperties
+		case "items":
+			converted[key] = openAPISchemaToJSONSchema(item, depth+1, dropped)
+		case "anyOf":
+			variants, isArray := item.([]any)
+			if !isArray {
+				converted[key] = item
+				continue
+			}
+			convertedVariants := make([]any, len(variants))
+			for index, variant := range variants {
+				convertedVariants[index] = openAPISchemaToJSONSchema(variant, depth+1, dropped)
+			}
+			converted[key] = convertedVariants
+		default:
+			if _, isInt64 := geminiSchemaInt64Fields[key]; isInt64 {
+				if text, isString := item.(string); isString {
+					if number, err := strconv.ParseInt(strings.TrimSpace(text), 10, 64); err == nil {
+						converted[key] = number
+						continue
+					}
+				}
+			}
+			converted[key] = item
+		}
+	}
+	if !nullable {
+		return converted
+	}
+	switch typed := converted["type"].(type) {
+	case string:
+		converted["type"] = []any{typed, "null"}
+	default:
+		if variants, isArray := converted["anyOf"].([]any); isArray {
+			converted["anyOf"] = append(variants, map[string]any{"type": "null"})
+		}
+	}
+	if values, isArray := converted["enum"].([]any); isArray && !slices.Contains(values, nil) {
+		converted["enum"] = append(values, nil)
+	}
+	return converted
+}