feat: publish sanitized custom.97 source
This commit is contained in:
1022 files changed
+69903
-19207
No files matched your search
@@ -42,7 +42,7 @@ func TestAlibabaResponsesProtocol(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("selects image and video usage profiles for every declared model", func(t *testing.T) {
|
||||
require.Len(t, plugin.Meta.UsageProfiles, 2)
|
||||
require.Len(t, plugin.Meta.UsageProfiles, 9)
|
||||
covered := make(map[string]bool)
|
||||
for _, profile := range plugin.Meta.UsageProfiles {
|
||||
for _, name := range profile.Models {
|
||||
@@ -54,13 +54,30 @@ func TestAlibabaResponsesProtocol(t *testing.T) {
|
||||
require.True(t, covered[name], name)
|
||||
schema, examples := plugin.Meta.UsageForModel(name)
|
||||
assert.Empty(t, examples)
|
||||
isImage := strings.Contains(name, "-image") || strings.Contains(name, "-t2i")
|
||||
if isImage {
|
||||
isImage := strings.Contains(name, "-image") || strings.Contains(name, "-t2i") || strings.Contains(name, "-i2i")
|
||||
switch {
|
||||
case strings.HasPrefix(name, "qwen-image-3.0"):
|
||||
assert.ElementsMatch(t, []string{"image_count", "output_image_type", "input_image_count"}, keysOf(schema), name)
|
||||
assert.Equal(t, plugin.Meta.UsageSchema["image_count"], schema["image_count"], name)
|
||||
assert.Equal(t, []string{"qima_output_1k", "qima_output_2k"}, schema["output_image_type"].Enum, name)
|
||||
case name == "z-image-turbo":
|
||||
assert.ElementsMatch(t, []string{"image_count", "prompt_extend"}, keysOf(schema), name)
|
||||
assert.Equal(t, "boolean", schema["prompt_extend"].Type, name)
|
||||
case isImage:
|
||||
assert.Equal(t, map[string]jsplugin.UsageFieldSchema{"image_count": plugin.Meta.UsageSchema["image_count"]}, schema, name)
|
||||
} else {
|
||||
assert.Equal(t, map[string]jsplugin.UsageFieldSchema{
|
||||
"seconds": plugin.Meta.UsageSchema["seconds"], "resolution": plugin.Meta.UsageSchema["resolution"],
|
||||
}, schema, name)
|
||||
case name == "wan2.6-i2v-flash":
|
||||
assert.ElementsMatch(t, []string{"seconds", "resolution", "audio"}, keysOf(schema), name)
|
||||
assert.Equal(t, plugin.Meta.UsageSchema["seconds"], schema["seconds"], name)
|
||||
assert.Equal(t, []string{"720P", "1080P"}, schema["resolution"].Enum, name)
|
||||
assert.Equal(t, "boolean", schema["audio"].Type, name)
|
||||
default:
|
||||
assert.ElementsMatch(t, []string{"seconds", "resolution"}, keysOf(schema), name)
|
||||
assert.Equal(t, plugin.Meta.UsageSchema["seconds"], schema["seconds"], name)
|
||||
assert.Subset(t, plugin.Meta.UsageSchema["resolution"].Enum, schema["resolution"].Enum, name)
|
||||
}
|
||||
for key, field := range schema {
|
||||
assert.NotEmpty(t, field.Description["en"], name+" "+key)
|
||||
assert.NotEmpty(t, field.Description["zh"], name+" "+key)
|
||||
}
|
||||
}
|
||||
})
|
||||
@@ -378,7 +395,14 @@ func TestAlibabaWanModelCapabilities(t *testing.T) {
|
||||
request := alibabaObject(t, value)["requestBody"].(map[string]any)
|
||||
body, facts, url := submitAlibabaRequest(t, plugin, tc.model, request)
|
||||
assert.Equal(t, "https://dashscope.aliyuncs.com/api/v1/services/aigc/"+tc.service+"/video-synthesis", url)
|
||||
assert.Equal(t, map[string]any{"seconds": tc.seconds, "resolution": tc.resolution}, facts)
|
||||
// The pricing table lists exactly the resolutions the model accepts.
|
||||
schema, _ := plugin.Meta.UsageForModel(tc.model)
|
||||
assert.Equal(t, tc.allowed, schema["resolution"].Enum)
|
||||
wantFacts := map[string]any{"seconds": tc.seconds, "resolution": tc.resolution}
|
||||
if tc.model == "wan2.6-i2v-flash" {
|
||||
wantFacts["audio"] = true // priced separately for silent output; defaults to audio
|
||||
}
|
||||
assert.Equal(t, wantFacts, facts)
|
||||
assert.Equal(t, tc.input, body["input"])
|
||||
parameters := body["parameters"].(map[string]any)
|
||||
if tc.size != "" {
|
||||
@@ -447,6 +471,36 @@ func TestAlibabaWanCompatibilityParameters(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAlibabaWanFlashAudioFact(t *testing.T) {
|
||||
plugin := newAlibabaPlugin(t)
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
request map[string]any
|
||||
audio bool
|
||||
}{
|
||||
{"audio defaults to true", map[string]any{}, true},
|
||||
{"explicit audio", map[string]any{"audio": true}, true},
|
||||
{"silent output", map[string]any{"audio": false}, false},
|
||||
{"native parameters win", map[string]any{"audio": true, "metadata": map[string]any{"parameters": map[string]any{"audio": false}}}, false},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
request := map[string]any{"model": "wan2.6-i2v-flash", "prompt": "a cat", "image": "https://cdn.example/first.png", "resolution": "720P", "duration": float64(5)}
|
||||
maps.Copy(request, tc.request)
|
||||
body, facts, _ := submitAlibabaRequest(t, plugin, "wan2.6-i2v-flash", request)
|
||||
assert.Equal(t, map[string]any{"seconds": float64(5), "resolution": "720P", "audio": tc.audio}, facts)
|
||||
parameters := body["parameters"].(map[string]any)
|
||||
if _, explicit := tc.request["audio"]; explicit || tc.request["metadata"] != nil {
|
||||
assert.Equal(t, tc.audio, parameters["audio"])
|
||||
} else {
|
||||
assert.NotContains(t, parameters, "audio")
|
||||
}
|
||||
})
|
||||
}
|
||||
// Other Wan models keep their two-fact schema even when a client sends audio.
|
||||
_, facts, _ := submitAlibabaRequest(t, plugin, "wan2.6-i2v", map[string]any{"model": "wan2.6-i2v", "prompt": "a cat", "image": "https://cdn.example/first.png", "audio": false, "duration": float64(5)})
|
||||
assert.Equal(t, map[string]any{"seconds": float64(5), "resolution": "1080P"}, facts)
|
||||
}
|
||||
|
||||
func TestAlibabaWanEntrypointsPreserveOptions(t *testing.T) {
|
||||
plugin := newAlibabaPlugin(t)
|
||||
input := map[string]any{"prompt": "a cat", "negative_prompt": "blur", "audio_url": "https://cdn.example/audio.mp3"}
|
||||
@@ -611,6 +665,9 @@ func TestAlibabaWanCompletionFactsAndArtifacts(t *testing.T) {
|
||||
{"Wan3 without reference video", "wan3.0-video-prime", map[string]any{"input_video_duration": 0, "output_video_duration": 8, "SR": 1080}, map[string]any{"seconds": float64(8), "resolution": "1080P"}},
|
||||
{"Wan3 missing input duration preserves reserved seconds", "wan3.0-video", map[string]any{"output_video_duration": 6, "SR": 720}, map[string]any{"resolution": "720P"}},
|
||||
{"2.7 continuation uses total duration", "wan2.7-i2v", map[string]any{"duration": 15, "output_video_duration": 12, "SR": 1080}, map[string]any{"seconds": float64(15), "resolution": "1080P"}},
|
||||
{"flash silent output settles as silent", "wan2.6-i2v-flash", map[string]any{"duration": 5, "SR": 720, "audio": false}, map[string]any{"seconds": float64(5), "resolution": "720P", "audio": false}},
|
||||
{"flash without audio flag keeps the estimate", "wan2.6-i2v-flash", map[string]any{"duration": 5, "SR": 1080}, map[string]any{"seconds": float64(5), "resolution": "1080P"}},
|
||||
{"audio flag is ignored for models priced without it", "wan2.6-i2v", map[string]any{"duration": 5, "SR": 1080, "audio": false}, map[string]any{"seconds": float64(5), "resolution": "1080P"}},
|
||||
{"missing usage keeps reservation", "wan2.7-t2v", map[string]any{}, nil},
|
||||
{"oversized duration rejected by host", "wan2.2-s2v", map[string]any{"duration": 1e30, "SR": 480}, nil},
|
||||
{"negative input cannot reduce Wan3 charge", "wan3.0-video", map[string]any{"input_video_duration": -5, "output_video_duration": 10}, nil},
|
||||
@@ -746,7 +803,7 @@ func TestAlibabaImageValidation(t *testing.T) {
|
||||
{"upstream SSE", "wan2.6-image", map[string]any{"metadata": map[string]any{"parameters": map[string]any{"stream": true}}}, "upstream streaming requires"},
|
||||
{"legacy synchronous", "wan2.2-t2i-flash", map[string]any{"metadata": map[string]any{"upstream_mode": "sync"}}, "only supports asynchronous"},
|
||||
{"edit without reference", "wan2.6-image", nil, "requires a reference image"},
|
||||
{"t2i with reference", "wan2.6-t2i", map[string]any{"image": "https://cdn.example/input.png"}, "too many input images"},
|
||||
{"t2i with reference", "wan2.6-t2i", map[string]any{"image": "https://cdn.example/input.png"}, "only supports text-to-image input"},
|
||||
{"invalid 4K", "wan2.7-image", map[string]any{"size": "4K"}, "4K is only supported"},
|
||||
{"oversized pixel dimensions", "wan2.7-image", map[string]any{"size": "999999999999*999999999999"}, "pixel and aspect-ratio limits"},
|
||||
{"4K group dimensions", "wan2.7-image-pro", map[string]any{"size": "4096*4096", "enable_sequential": true}, "pixel and aspect-ratio limits"},
|
||||
@@ -807,12 +864,12 @@ func TestAlibabaImageResults(t *testing.T) {
|
||||
require.Len(t, artifacts, 2)
|
||||
assert.Equal(t, "image-2", artifacts[1].Key)
|
||||
assert.Equal(t, "image", artifacts[1].Type)
|
||||
content, err := adaptor.BuildContentRequest(task, "image-2", relaychannel.TaskArtifactClientRequest{Method: http.MethodGet})
|
||||
content, err := adaptor.BuildContentRequest(t.Context(), task, "image-2", relaychannel.TaskArtifactClientRequest{Method: http.MethodGet})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, second, content.URL)
|
||||
assert.True(t, content.Credentialless)
|
||||
assert.Empty(t, content.Headers)
|
||||
_, err = adaptor.BuildContentRequest(task, "image-3", relaychannel.TaskArtifactClientRequest{Method: http.MethodGet})
|
||||
_, err = adaptor.BuildContentRequest(t.Context(), task, "image-3", relaychannel.TaskArtifactClientRequest{Method: http.MethodGet})
|
||||
require.Error(t, err)
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_responses", "renderFinal"}, map[string]any{
|
||||
"artifacts": map[string]any{"image-1": map[string]any{"url": "/v1/tasks/task_public/artifacts/image-1/content"}, "image-2": map[string]any{"url": "/v1/tasks/task_public/artifacts/image-2/content"}},
|
||||
@@ -975,3 +1032,358 @@ func TestAlibabaSynchronousMultiImageAndStream(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// The OpenAI Images protocol claims every declared image model on both image
|
||||
// endpoints; chat and video models keep the built-in relay.
|
||||
func TestAlibabaOpenAIImageProtocolBindings(t *testing.T) {
|
||||
source, err := builtinplugins.Source("alibaba")
|
||||
require.NoError(t, err)
|
||||
registry := jsplugin.NewRegistry()
|
||||
plugin, err := registry.RegisterFactory(source, jsplugin.Options{Key: "alibaba"})
|
||||
require.NoError(t, err)
|
||||
for _, name := range []string{"qwen-image-3.0-pro", "qwen-image-plus", "qwen-image-edit-plus", "z-image-turbo", "wan2.6-t2i", "wan2.2-t2i-flash", "wan2.5-i2i-preview", "wanx2.1-imageedit", "qwen-image-edit-max-2026-01-16"} {
|
||||
for _, path := range []string{"/v1/images/generations", "/v1/images/edits"} {
|
||||
binding, found := registry.Generation().LookupEndpoint(http.MethodPost, path, name)
|
||||
require.True(t, found, name+" "+path)
|
||||
assert.Same(t, plugin, binding.Plugin)
|
||||
assert.Equal(t, "openai_image", binding.Protocol)
|
||||
}
|
||||
_, found := registry.Generation().LookupEndpoint(http.MethodPost, "/v1/videos", name)
|
||||
assert.False(t, found, name)
|
||||
}
|
||||
for _, name := range []string{"wan2.6-t2v", "qwen-turbo"} {
|
||||
_, found := registry.Generation().LookupEndpoint(http.MethodPost, "/v1/images/generations", name)
|
||||
assert.False(t, found, name)
|
||||
}
|
||||
}
|
||||
|
||||
func decodeAlibabaImage(t *testing.T, plugin *jsplugin.LoadedPlugin, operation string, body map[string]any) (map[string]any, error) {
|
||||
t.Helper()
|
||||
path := "/v1/images/generations"
|
||||
if operation == "edit" {
|
||||
path = "/v1/images/edits"
|
||||
}
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_image", "decodeRequest"}, map[string]any{
|
||||
"protocol": "openai_image", "operation": operation, "model": body["model"], "path": path, "method": http.MethodPost,
|
||||
"body": map[string]any{"kind": "json", "value": body},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return alibabaObject(t, value), nil
|
||||
}
|
||||
|
||||
// OpenAI image requests map onto the DashScope services the models support,
|
||||
// reserve the requested count, and reject counts the vendor would refuse
|
||||
// before any quota is reserved.
|
||||
func TestAlibabaOpenAIImageDecodeAndSubmission(t *testing.T) {
|
||||
plugin := newAlibabaPlugin(t)
|
||||
const base = "https://dashscope.aliyuncs.com/api/v1/services/aigc/"
|
||||
for _, tc := range []struct {
|
||||
name, model, operation string
|
||||
body map[string]any
|
||||
wantService, wantAsync string
|
||||
wantAction string
|
||||
wantN float64
|
||||
wantFacts map[string]any
|
||||
wantRatios map[string]any
|
||||
}{
|
||||
{"qwen-image-3.0-pro two images", "qwen-image-3.0-pro", "generate", map[string]any{"prompt": "a white siamese cat", "n": 2, "size": "1024x1024"},
|
||||
"multimodal-generation/generation", "", "text_to_image", 2,
|
||||
map[string]any{"image_count": float64(2), "output_image_type": "qima_output_1k", "input_image_count": float64(0)}, map[string]any{"image_count": float64(2)}},
|
||||
{"qwen-image-3.0-pro without size reserves the 2K tier", "qwen-image-3.0", "generate", map[string]any{"prompt": "poster"},
|
||||
"multimodal-generation/generation", "", "text_to_image", 1,
|
||||
map[string]any{"image_count": float64(1), "output_image_type": "qima_output_2k", "input_image_count": float64(0)}, map[string]any{"image_count": float64(1)}},
|
||||
{"fixed count model", "qwen-image-plus", "generate", map[string]any{"prompt": "a cat", "size": "1328x1328"},
|
||||
"multimodal-generation/generation", "", "text_to_image", 1, map[string]any{"image_count": float64(1)}, map[string]any{"image_count": float64(1)}},
|
||||
{"z-image with prompt rewriting", "z-image-turbo", "generate", map[string]any{"prompt": "a cat", "prompt_extend": true},
|
||||
"multimodal-generation/generation", "", "text_to_image", 1,
|
||||
map[string]any{"image_count": float64(1), "prompt_extend": true}, map[string]any{"image_count": float64(1), "prompt_extend_ratio": float64(2)}},
|
||||
{"legacy text-to-image is asynchronous", "wan2.2-t2i-flash", "generate", map[string]any{"prompt": "a cat", "n": 2, "size": "1024x1024"},
|
||||
"text2image/image-synthesis", "enable", "text_to_image", 2, map[string]any{"image_count": float64(2)}, map[string]any{"image_count": float64(2)}},
|
||||
{"wan2.6 text-to-image is asynchronous by default", "wan2.6-t2i", "generate", map[string]any{"prompt": "a cat", "n": 2},
|
||||
"image-generation/generation", "enable", "text_to_image", 2, map[string]any{"image_count": float64(2)}, map[string]any{"image_count": float64(2)}},
|
||||
{"qwen edit with an image URL", "qwen-image-edit-plus", "edit", map[string]any{"prompt": "watercolor", "image": "https://cdn.example/input.png"},
|
||||
"multimodal-generation/generation", "", "image_to_image", 1, map[string]any{"image_count": float64(1)}, map[string]any{"image_count": float64(1)}},
|
||||
{"qwen-image-3.0 edit counts input images", "qwen-image-3.0-pro", "edit", map[string]any{"prompt": "watercolor", "image": []any{"https://cdn.example/1.png", "https://cdn.example/2.png"}, "size": "2048x2048"},
|
||||
"multimodal-generation/generation", "", "image_to_image", 1,
|
||||
map[string]any{"image_count": float64(1), "output_image_type": "qima_output_2k", "input_image_count": float64(2)}, map[string]any{"image_count": float64(1)}},
|
||||
{"wan2.5 legacy edit is asynchronous", "wan2.5-i2i-preview", "edit", map[string]any{"prompt": "watercolor", "image": "https://cdn.example/input.png", "n": 2},
|
||||
"image2image/image-synthesis", "enable", "image_to_image", 2, map[string]any{"image_count": float64(2)}, map[string]any{"image_count": float64(2)}},
|
||||
{"wanx2.1 function edit is asynchronous", "wanx2.1-imageedit", "edit", map[string]any{"prompt": "make the hair red", "image": "https://cdn.example/input.png", "n": 2},
|
||||
"image2image/image-synthesis", "enable", "image_to_image", 2, map[string]any{"image_count": float64(2)}, map[string]any{"image_count": float64(2)}},
|
||||
{"provider parameters pass through", "qwen-image-2.0-pro", "generate", map[string]any{"prompt": "a cat", "n": 3, "parameters": map[string]any{"negative_prompt": "blurry", "watermark": false}},
|
||||
"multimodal-generation/generation", "", "text_to_image", 3, map[string]any{"image_count": float64(3)}, map[string]any{"image_count": float64(3)}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
body := map[string]any{"model": tc.model}
|
||||
maps.Copy(body, tc.body)
|
||||
resolved, err := decodeAlibabaImage(t, plugin, tc.operation, body)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "submit", resolved["kind"])
|
||||
assert.Equal(t, tc.model, resolved["model"])
|
||||
assert.Equal(t, tc.wantAction, resolved["action"])
|
||||
request := resolved["requestBody"].(map[string]any)
|
||||
upstreamBody, facts, url := submitAlibabaRequest(t, plugin, tc.model, request)
|
||||
assert.Equal(t, base+tc.wantService, url)
|
||||
assert.Equal(t, tc.model, upstreamBody["model"])
|
||||
parameters := upstreamBody["parameters"].(map[string]any)
|
||||
assert.Equal(t, tc.wantN, parameters["n"], "the reserved count is sent explicitly")
|
||||
if size, ok := tc.body["size"].(string); ok {
|
||||
assert.Equal(t, strings.Replace(size, "x", "*", 1), parameters["size"])
|
||||
}
|
||||
if extra, ok := tc.body["parameters"].(map[string]any); ok {
|
||||
for key, expected := range alibabaObject(t, extra) {
|
||||
assert.Equal(t, expected, parameters[key], key)
|
||||
}
|
||||
}
|
||||
assert.Equal(t, tc.wantFacts, facts)
|
||||
value, err := plugin.Engine.Call(t.Context(), "extractUsage", map[string]any{"upstreamModel": tc.model, "model": tc.model, "usagePurpose": "billing_ratios", "requestBody": request})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantRatios, alibabaObject(t, value))
|
||||
value, err = plugin.Engine.Call(t.Context(), "buildSubmitRequest", map[string]any{"upstreamModel": tc.model, "requestBody": request, "baseUrl": "https://dashscope.aliyuncs.com", "apiKey": "k"})
|
||||
require.NoError(t, err)
|
||||
headers := alibabaObject(t, value)["headers"].(map[string]any)
|
||||
if tc.wantAsync == "" {
|
||||
assert.NotContains(t, headers, "X-DashScope-Async")
|
||||
} else {
|
||||
assert.Equal(t, tc.wantAsync, headers["X-DashScope-Async"])
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("multipart edits address every uploaded file", func(t *testing.T) {
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_image", "decodeRequest"}, map[string]any{
|
||||
"protocol": "openai_image", "operation": "edit", "model": "qwen-image-edit-plus", "path": "/v1/images/edits", "method": http.MethodPost,
|
||||
"body": map[string]any{"kind": "multipart",
|
||||
"fields": map[string][]string{"model": {"qwen-image-edit-plus"}, "prompt": {"watercolor"}, "n": {"2"}, "response_format": {"b64_json"}, "watermark": {"false"}},
|
||||
"files": []any{
|
||||
map[string]any{"ref": "request_file:image[]", "field": "image[]", "filename": "a.png", "mimeType": "image/png", "size": 10},
|
||||
map[string]any{"ref": "request_file:image[]#1", "field": "image[]", "filename": "b.jpg", "mimeType": "image/jpeg", "size": 10},
|
||||
map[string]any{"ref": "request_file:mask", "field": "mask", "filename": "m.png", "mimeType": "image/png", "size": 10},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
resolved := alibabaObject(t, value)
|
||||
assert.Equal(t, "image_to_image", resolved["action"])
|
||||
request := resolved["requestBody"].(map[string]any)
|
||||
assert.Equal(t, float64(2), request["n"])
|
||||
assert.Equal(t, false, request["watermark"])
|
||||
images := request["images"].([]any)
|
||||
require.Len(t, images, 2, "mask uploads are not reference images")
|
||||
assert.Equal(t, map[string]any{"__fileRef": "request_file:image[]", "encoding": "dataUrl", "mimeType": "image/png", "maxBytes": float64(10485760)}, images[0])
|
||||
assert.Equal(t, "request_file:image[]#1", images[1].(map[string]any)["__fileRef"])
|
||||
driver := map[string]any{"upstreamModel": "qwen-image-edit-plus", "model": "qwen-image-edit-plus", "requestBody": request, "baseUrl": "https://dashscope.aliyuncs.com", "apiKey": "k", "usagePurpose": "facts"}
|
||||
value, err = plugin.Engine.Call(t.Context(), "buildSubmitRequest", driver)
|
||||
require.NoError(t, err)
|
||||
body := alibabaObject(t, value)["body"].(map[string]any)
|
||||
content := body["input"].(map[string]any)["messages"].([]any)[0].(map[string]any)["content"].([]any)
|
||||
require.Len(t, content, 3)
|
||||
assert.Equal(t, "request_file:image[]#1", content[1].(map[string]any)["image"].(map[string]any)["__fileRef"], "placeholders reach the upstream body for the host to inline")
|
||||
assert.Equal(t, "watercolor", content[2].(map[string]any)["text"])
|
||||
value, err = plugin.Engine.Call(t.Context(), "extractUsage", driver)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"image_count": float64(2)}, alibabaObject(t, value))
|
||||
})
|
||||
|
||||
// wanx2.1-imageedit selects its operation with input.function. OpenAI edit
|
||||
// fields map onto the documented body: the image becomes base_image_url, a
|
||||
// mask becomes mask_image_url and selects local repainting, and the native
|
||||
// route forwards the DashScope body unchanged.
|
||||
t.Run("wanx2.1-imageedit maps OpenAI edit fields onto the function API", func(t *testing.T) {
|
||||
const model = "wanx2.1-imageedit"
|
||||
const image, mask = "https://cdn.example/base.png", "https://cdn.example/mask.png"
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body map[string]any
|
||||
wantInput map[string]any
|
||||
wantParameters map[string]any
|
||||
}{
|
||||
{"defaults to instruction editing", map[string]any{"prompt": "make the hair red", "image": image},
|
||||
map[string]any{"prompt": "make the hair red", "function": "description_edit", "base_image_url": image}, map[string]any{"n": float64(1)}},
|
||||
{"a mask selects local repainting", map[string]any{"prompt": "a ceramic rabbit", "image": image, "mask": mask, "n": 2},
|
||||
map[string]any{"prompt": "a ceramic rabbit", "function": "description_edit_with_mask", "base_image_url": image, "mask_image_url": mask}, map[string]any{"n": float64(2)}},
|
||||
{"explicit function with its parameters", map[string]any{"prompt": "a green fairy", "image": image, "function": "expand", "top_scale": 1.5, "watermark": true},
|
||||
map[string]any{"prompt": "a green fairy", "function": "expand", "base_image_url": image}, map[string]any{"n": float64(1), "top_scale": 1.5, "watermark": true}},
|
||||
} {
|
||||
body := map[string]any{"model": model}
|
||||
maps.Copy(body, tc.body)
|
||||
resolved, err := decodeAlibabaImage(t, plugin, "edit", body)
|
||||
require.NoError(t, err, tc.name)
|
||||
upstreamBody, facts, url := submitAlibabaRequest(t, plugin, model, resolved["requestBody"].(map[string]any))
|
||||
assert.Equal(t, base+"image2image/image-synthesis", url, tc.name)
|
||||
assert.Equal(t, tc.wantInput, upstreamBody["input"], tc.name)
|
||||
assert.Equal(t, tc.wantParameters, upstreamBody["parameters"], tc.name)
|
||||
assert.Equal(t, map[string]any{"image_count": tc.wantParameters["n"]}, facts, tc.name)
|
||||
}
|
||||
|
||||
native, err := plugin.Engine.CallPath(t.Context(), "native", []string{"createImageTask"}, map[string]any{
|
||||
"path": "/ali/api/v1/services/aigc/image2image/image-synthesis",
|
||||
"body": map[string]any{"kind": "json", "value": map[string]any{"model": model, "input": map[string]any{"function": "super_resolution", "prompt": "sharpen", "base_image_url": image}, "parameters": map[string]any{"upscale_factor": 2, "n": 1}}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
upstreamBody, facts, url := submitAlibabaRequest(t, plugin, model, alibabaObject(t, native)["requestBody"].(map[string]any))
|
||||
assert.Equal(t, base+"image2image/image-synthesis", url)
|
||||
assert.Equal(t, map[string]any{"function": "super_resolution", "prompt": "sharpen", "base_image_url": image}, upstreamBody["input"])
|
||||
assert.Equal(t, map[string]any{"upscale_factor": float64(2), "n": float64(1)}, upstreamBody["parameters"])
|
||||
assert.Equal(t, map[string]any{"image_count": float64(1)}, facts)
|
||||
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_image", "decodeRequest"}, map[string]any{
|
||||
"protocol": "openai_image", "operation": "edit", "model": model, "path": "/v1/images/edits", "method": http.MethodPost,
|
||||
"body": map[string]any{"kind": "multipart",
|
||||
"fields": map[string][]string{"model": {model}, "prompt": {"a ceramic rabbit"}},
|
||||
"files": []any{
|
||||
map[string]any{"ref": "request_file:image", "field": "image", "filename": "a.png", "mimeType": "image/png", "size": 10},
|
||||
map[string]any{"ref": "request_file:mask", "field": "mask", "filename": "m.png", "mimeType": "image/png", "size": 10},
|
||||
},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
value, err = plugin.Engine.Call(t.Context(), "buildSubmitRequest", map[string]any{"upstreamModel": model, "requestBody": alibabaObject(t, value)["requestBody"], "baseUrl": "https://dashscope.aliyuncs.com", "apiKey": "k"})
|
||||
require.NoError(t, err)
|
||||
input := alibabaObject(t, value)["body"].(map[string]any)["input"].(map[string]any)
|
||||
assert.Equal(t, "description_edit_with_mask", input["function"], "a mask upload selects local repainting")
|
||||
assert.Equal(t, "request_file:image", input["base_image_url"].(map[string]any)["__fileRef"])
|
||||
assert.Equal(t, "request_file:mask", input["mask_image_url"].(map[string]any)["__fileRef"], "the mask placeholder reaches the upstream body for the host to inline")
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
name, model, operation string
|
||||
body map[string]any
|
||||
decodeErr, submitErr string
|
||||
}{
|
||||
{"wanx2.1-imageedit takes exactly one image", "wanx2.1-imageedit", "edit", map[string]any{"prompt": "x", "image": []any{"https://cdn.example/1.png", "https://cdn.example/2.png"}}, "", "exactly one input image"},
|
||||
{"wanx2.1-imageedit rejects unknown functions", "wanx2.1-imageedit", "edit", map[string]any{"prompt": "x", "image": "https://cdn.example/1.png", "function": "paint"}, "", "function must be one of"},
|
||||
{"wanx2.1-imageedit local repainting needs a mask", "wanx2.1-imageedit", "edit", map[string]any{"prompt": "x", "image": "https://cdn.example/1.png", "function": "description_edit_with_mask"}, "", "requires a mask image"},
|
||||
{"wanx2.1-imageedit has no size parameter", "wanx2.1-imageedit", "edit", map[string]any{"prompt": "x", "image": "https://cdn.example/1.png", "size": "1024x1024"}, "", "does not accept size"},
|
||||
{"wanx2.1-imageedit bounds n", "wanx2.1-imageedit", "edit", map[string]any{"prompt": "x", "image": "https://cdn.example/1.png", "n": 5}, "", "n must be an integer between 1 and 4"},
|
||||
{"fixed count model rejects n=2 before reservation", "qwen-image-plus", "generate", map[string]any{"prompt": "a cat", "n": 2}, "", "n must be 1 for this model"},
|
||||
{"z-image rejects n=2", "z-image-turbo", "generate", map[string]any{"prompt": "a cat", "n": 2}, "", "n must be 1 for this model"},
|
||||
{"qwen-image-3.0 bounds n", "qwen-image-3.0-pro", "generate", map[string]any{"prompt": "a cat", "n": 7}, "", "n must be an integer between 1 and 6"},
|
||||
{"legacy bounds n", "wan2.2-t2i-flash", "generate", map[string]any{"prompt": "a cat", "n": 5}, "", "n must be an integer between 1 and 4"},
|
||||
{"edit requires an image", "qwen-image-edit-plus", "edit", map[string]any{"prompt": "watercolor"}, "image is required", ""},
|
||||
{"edit-only model requires an image on generations", "qwen-image-edit", "generate", map[string]any{"prompt": "watercolor"}, "", "requires at least one input image"},
|
||||
{"text-to-image model rejects images", "qwen-image-plus", "edit", map[string]any{"prompt": "a cat", "image": "https://cdn.example/1.png"}, "", "only supports text-to-image input"},
|
||||
{"streaming is rejected", "qwen-image-plus", "generate", map[string]any{"prompt": "a cat", "stream": true}, "stream is not supported", ""},
|
||||
{"zero count is rejected", "qwen-image-3.0-pro", "generate", map[string]any{"prompt": "a cat", "n": 0}, "n must be a positive integer", ""},
|
||||
{"response_format is validated", "qwen-image-plus", "generate", map[string]any{"prompt": "a cat", "response_format": "png"}, "response_format must be url or b64_json", ""},
|
||||
{"prompt is required", "qwen-image-plus", "generate", map[string]any{}, "prompt is required", ""},
|
||||
{"size bounds follow the model", "z-image-turbo", "generate", map[string]any{"prompt": "a cat", "size": "256x256"}, "", "pixel and aspect-ratio limits"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
body := map[string]any{"model": tc.model}
|
||||
maps.Copy(body, tc.body)
|
||||
resolved, err := decodeAlibabaImage(t, plugin, tc.operation, body)
|
||||
if tc.decodeErr != "" {
|
||||
require.ErrorContains(t, err, tc.decodeErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
request := resolved["requestBody"].(map[string]any)
|
||||
_, err = plugin.Engine.Call(t.Context(), "buildSubmitRequest", map[string]any{"requestBody": request, "upstreamModel": tc.model})
|
||||
require.ErrorContains(t, err, tc.submitErr)
|
||||
info := &relaycommon.RelayInfo{ChannelMeta: &relaycommon.ChannelMeta{UpstreamModelName: tc.model}, TaskRelayInfo: &relaycommon.TaskRelayInfo{}}
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(info)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/images/generations", nil)
|
||||
c.Set("task_request", request)
|
||||
taskErr := adaptor.ValidateRequestAndSetAction(c, info)
|
||||
require.NotNil(t, taskErr, "the production validator must reject the request before billing")
|
||||
assert.Equal(t, http.StatusBadRequest, taskErr.StatusCode)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Count derivation for synchronous image results: usage.image_count first,
|
||||
// then the Qwen-Image-3.0 usage.output_image_count, then image payloads
|
||||
// flattened across every choice; payload-less parts never count and an
|
||||
// inconsistent count retains the reservation. The response renderer emits one
|
||||
// OpenAI data entry per image, so the client sees as many images as are billed.
|
||||
func TestAlibabaOpenAIImageCountDerivationAndRender(t *testing.T) {
|
||||
plugin := newAlibabaPlugin(t)
|
||||
const first, second = "https://cdn.example/first.png", "https://cdn.example/second.png"
|
||||
twoImagesOneChoice := []any{map[string]any{"finish_reason": "stop", "message": map[string]any{"role": "assistant", "content": []any{map[string]any{"image": first}, map[string]any{"image": second}}}}}
|
||||
for _, tc := range []struct {
|
||||
name, model string
|
||||
output map[string]any
|
||||
usage map[string]any
|
||||
wantFacts map[string]any
|
||||
wantData int
|
||||
wantErr bool
|
||||
}{
|
||||
{"qwen-image-3.0 reports output_image_count", "qwen-image-3.0-pro", map[string]any{"choices": twoImagesOneChoice},
|
||||
map[string]any{"output_width": 1024, "output_height": 1024, "input_image_count": 0, "input_image_type": "qima_input_1k", "output_image_count": 2, "output_image_type": "qima_output_1k"},
|
||||
map[string]any{"image_count": float64(2), "output_image_type": "qima_output_1k", "input_image_count": float64(0)}, 2, false},
|
||||
{"one choice with two images and no usage counts payloads", "qwen-image-2.0-pro", map[string]any{"choices": twoImagesOneChoice}, nil,
|
||||
map[string]any{"image_count": float64(2)}, 2, false},
|
||||
{"usage.image_count is preferred", "wan2.6-t2i", map[string]any{"choices": twoImagesOneChoice}, map[string]any{"image_count": 2},
|
||||
map[string]any{"image_count": float64(2)}, 2, false},
|
||||
{"payload-less parts are not counted", "qwen-image-plus", map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": []any{map[string]any{"text": "revised"}, map[string]any{"image": first}, map[string]any{"image": ""}}}}}}, nil,
|
||||
map[string]any{"image_count": float64(1)}, 1, false},
|
||||
{"z-image keeps its count", "z-image-turbo", map[string]any{"choices": []any{map[string]any{"message": map[string]any{"content": []any{map[string]any{"image": first}}}}}}, map[string]any{"image_count": 1, "width": 1024, "height": 1536},
|
||||
map[string]any{"image_count": float64(1)}, 1, false},
|
||||
{"inconsistent zero count retains the reservation", "qwen-image-3.0-pro", map[string]any{"choices": twoImagesOneChoice}, map[string]any{"output_image_count": 0},
|
||||
nil, 2, true},
|
||||
{"fractional count retains the reservation", "wan2.6-t2i", map[string]any{"choices": twoImagesOneChoice}, map[string]any{"image_count": 1.5},
|
||||
nil, 2, true},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
body := map[string]any{"request_id": "r", "output": tc.output}
|
||||
if tc.usage != nil {
|
||||
body["usage"] = tc.usage
|
||||
}
|
||||
request := map[string]any{"model": tc.model, "prompt": "a cat", "n": 2}
|
||||
if tc.model == "qwen-image-plus" || tc.model == "z-image-turbo" {
|
||||
request["n"] = 1
|
||||
}
|
||||
if strings.HasPrefix(tc.model, "wan") {
|
||||
// Wan models default to the asynchronous service; the synchronous response shape needs the sync mode.
|
||||
request["metadata"] = map[string]any{"upstream_mode": "sync"}
|
||||
}
|
||||
info := &relaycommon.RelayInfo{OriginModelName: tc.model, ChannelMeta: &relaycommon.ChannelMeta{UpstreamModelName: tc.model, ChannelBaseUrl: "https://dashscope.aliyuncs.com"}, TaskRelayInfo: &relaycommon.TaskRelayInfo{PublicTaskID: "task_public"}}
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(info)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/images/generations", nil)
|
||||
c.Set("task_request", request)
|
||||
require.Nil(t, adaptor.ValidateRequestAndSetAction(c, info))
|
||||
encoded, err := common.Marshal(body)
|
||||
require.NoError(t, err)
|
||||
parsed, taskErr := adaptor.ParseResponse(c, &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}}, Body: io.NopCloser(bytes.NewReader(encoded))}, info)
|
||||
require.Nil(t, taskErr)
|
||||
require.NotNil(t, parsed.Immediate)
|
||||
assert.Equal(t, "SUCCESS", parsed.Immediate.Status)
|
||||
assert.Equal(t, "task_public", parsed.UpstreamTaskID, "synchronous results keep the host task id")
|
||||
if tc.wantErr {
|
||||
assert.Nil(t, parsed.Immediate.UsageFacts, "an unrecognized count must not replace the reservation")
|
||||
} else {
|
||||
assert.Equal(t, tc.wantFacts, parsed.Immediate.UsageFacts)
|
||||
ratios := adaptor.AdjustBillingOnSubmit(info, parsed.TaskData)
|
||||
assert.Equal(t, tc.wantFacts["image_count"], ratios["image_count"], "legacy per-call pricing settles to the same count")
|
||||
}
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_image", "render"}, map[string]any{"model": tc.model}, map[string]any{"task_id": "task_public", "status": "SUCCESS", "created_at": 1700000000, "data": body})
|
||||
require.NoError(t, err)
|
||||
rendered := alibabaObject(t, value)
|
||||
assert.Equal(t, float64(1700000000), rendered["created"])
|
||||
data := rendered["data"].([]any)
|
||||
require.Len(t, data, tc.wantData)
|
||||
assert.Equal(t, first, data[0].(map[string]any)["url"])
|
||||
assert.NotContains(t, data[0], "b64_json")
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("legacy results and Base64 payloads render as OpenAI entries", func(t *testing.T) {
|
||||
body := map[string]any{"output": map[string]any{"task_status": "SUCCEEDED", "results": []any{map[string]any{"url": first}, map[string]any{"code": "DataInspectionFailed"}, map[string]any{"url": "data:image/png;base64,QUFB"}}}, "usage": map[string]any{"image_count": 2}}
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_image", "render"}, map[string]any{}, map[string]any{"status": "SUCCESS", "created_at": 1, "data": body})
|
||||
require.NoError(t, err)
|
||||
rendered := alibabaObject(t, value)
|
||||
data := rendered["data"].([]any)
|
||||
require.Len(t, data, 2)
|
||||
assert.Equal(t, map[string]any{"url": first}, data[0])
|
||||
assert.Equal(t, map[string]any{"b64_json": "QUFB"}, data[1])
|
||||
// metadata keeps the upstream body so clients still see DashScope usage.
|
||||
assert.Equal(t, float64(2), rendered["metadata"].(map[string]any)["usage"].(map[string]any)["image_count"])
|
||||
})
|
||||
}
|
||||
@@ -38,7 +38,7 @@ func TestAlibabaWan3(t *testing.T) {
|
||||
return roundTrip(t, value), nil
|
||||
}
|
||||
|
||||
t.Run("duration -1 becomes the auto_duration marker so the host accepts the body", func(t *testing.T) {
|
||||
t.Run("duration -1 becomes the auto_duration marker in every decoder", func(t *testing.T) {
|
||||
resolved, callErr := decodeResponses("wan3.0-video", map[string]any{"model": "wan3.0-video", "input": "a cat", "duration": -1})
|
||||
require.NoError(t, callErr)
|
||||
requestBody := resolved["requestBody"].(map[string]any)
|
||||
@@ -83,6 +83,16 @@ func TestAlibabaWan3(t *testing.T) {
|
||||
require.ErrorContains(t, callErr, "only supported by wan3.0")
|
||||
})
|
||||
|
||||
// /v1/video/generations has no plugin decoder, so -1 reaches the host's
|
||||
// request validation as a vendor parameter; duration-auto@1 lets it through.
|
||||
t.Run("metadata parameters keep -1 on /v1/video/generations", func(t *testing.T) {
|
||||
body, facts, _ := submitAlibabaRequest(t, plugin, "wan3.0-video", map[string]any{
|
||||
"model": "wan3.0-video", "prompt": "a cat", "metadata": map[string]any{"parameters": map[string]any{"duration": -1}},
|
||||
})
|
||||
assert.Equal(t, float64(-1), body["parameters"].(map[string]any)["duration"])
|
||||
assert.Equal(t, float64(30), facts["seconds"])
|
||||
})
|
||||
|
||||
t.Run("channel-mapped alias resolves defaults from the upstream model", func(t *testing.T) {
|
||||
direct, callErr := plugin.Engine.Call(t.Context(), "buildSubmitRequest", submitCtx("wan3.0-video", "wan3.0-video", map[string]any{"model": "wan3.0-video", "prompt": "a cat"}))
|
||||
require.NoError(t, callErr)
|
||||
|
||||
@@ -2,6 +2,7 @@ package plugins
|
||||
|
||||
import (
|
||||
"io/fs"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
@@ -10,7 +11,7 @@ import (
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
var expectedKeys = []string{"alibaba", "doubao", "google", "hailuo", "jimeng", "kling", "sora", "sunoapi", "vertex-ai", "vidu"}
|
||||
var expectedKeys = []string{"alibaba", "doubao", "google", "hailuo", "jimeng", "kling", "sora", "sunoapi", "vertex-ai", "vidu", "xai"}
|
||||
|
||||
func TestBuiltInVendorPluginsDeclareNativeRoutesAndLegacyChannelTypes(t *testing.T) {
|
||||
generation := jsplugin.DefaultRegistry.Generation()
|
||||
@@ -34,6 +35,23 @@ func TestBuiltInVendorPluginsDeclareNativeRoutesAndLegacyChannelTypes(t *testing
|
||||
{"GET", "/suno/fetch/:task_id", "sunoapi", jsplugin.RouteTypeQuery, "", "renderTask"},
|
||||
{"POST", "/doubao/api/v3/contents/generations/tasks", "doubao", jsplugin.RouteTypeSubmit, "", "taskCreated"},
|
||||
{"GET", "/doubao/api/v3/contents/generations/tasks/:task_id", "doubao", jsplugin.RouteTypeQuery, "", "taskStatus"},
|
||||
{"POST", "/doubao/api/v3/images/generations", "doubao", jsplugin.RouteTypeSubmit, "", "imageCreated"},
|
||||
{"POST", "/xai/v1/videos/generations", "xai", jsplugin.RouteTypeSubmit, "", "videoCreated"},
|
||||
{"POST", "/xai/v1/videos/edits", "xai", jsplugin.RouteTypeSubmit, "", "videoCreated"},
|
||||
{"POST", "/xai/v1/videos/extensions", "xai", jsplugin.RouteTypeSubmit, "", "videoCreated"},
|
||||
{"GET", "/xai/v1/videos/:request_id", "xai", jsplugin.RouteTypeQuery, "", "videoStatus"},
|
||||
{"POST", "/hailuo/v1/video_generation", "hailuo", jsplugin.RouteTypeSubmit, "", "taskCreated"},
|
||||
{"GET", "/hailuo/v1/query/video_generation/:task_id", "hailuo", jsplugin.RouteTypeQuery, "", "taskStatus"},
|
||||
{"POST", "/hailuo/v2/video_generation", "hailuo", jsplugin.RouteTypeSubmit, "", "taskCreated"},
|
||||
{"GET", "/hailuo/v2/query/video_generation/:task_id", "hailuo", jsplugin.RouteTypeQuery, "", "h3TaskStatus"},
|
||||
{"POST", "/vidu/ent/v2/text2video", "vidu", jsplugin.RouteTypeSubmit, "", "taskCreated"},
|
||||
{"POST", "/vidu/ent/v2/img2video", "vidu", jsplugin.RouteTypeSubmit, "", "taskCreated"},
|
||||
{"POST", "/vidu/ent/v2/start-end2video", "vidu", jsplugin.RouteTypeSubmit, "", "taskCreated"},
|
||||
{"POST", "/vidu/ent/v2/reference2video", "vidu", jsplugin.RouteTypeSubmit, "", "taskCreated"},
|
||||
{"GET", "/vidu/ent/v2/tasks/:task_id/creations", "vidu", jsplugin.RouteTypeQuery, "", "taskStatus"},
|
||||
{"POST", "/google/v1beta/models/:model", "google", jsplugin.RouteTypeSubmit, "", "operation"},
|
||||
{"GET", "/google/v1beta/models/:model/operations/:task_id", "google", jsplugin.RouteTypeQuery, "", "operation"},
|
||||
{"POST", "/vertex-ai/v1/projects/:project/locations/:location/publishers/google/models/:model", "vertex-ai", jsplugin.RouteTypeDynamic, "", "operation"},
|
||||
}
|
||||
for _, expected := range routes {
|
||||
t.Run(expected.method+" "+expected.path, func(t *testing.T) {
|
||||
@@ -46,6 +64,13 @@ func TestBuiltInVendorPluginsDeclareNativeRoutesAndLegacyChannelTypes(t *testing
|
||||
})
|
||||
}
|
||||
|
||||
// The legacy Wan edit route serves both the wan2.5 image-list body and the
|
||||
// wanx2.1-imageedit function API.
|
||||
imageEdit, found := generation.LookupDeclaredRoute("POST", "/ali/api/v1/services/aigc/image2image/image-synthesis")
|
||||
require.True(t, found)
|
||||
assert.Equal(t, "alibaba", imageEdit.Plugin.Meta.Key)
|
||||
assert.ElementsMatch(t, []string{"wan2.5-i2i-preview", "wanx2.1-imageedit"}, imageEdit.Route.Models)
|
||||
|
||||
channelTypes := []struct {
|
||||
value int
|
||||
key string
|
||||
@@ -57,6 +82,7 @@ func TestBuiltInVendorPluginsDeclareNativeRoutesAndLegacyChannelTypes(t *testing
|
||||
{51, "jimeng"},
|
||||
{54, "doubao"},
|
||||
{55, "sora"},
|
||||
{48, "xai"},
|
||||
}
|
||||
for _, channelType := range channelTypes {
|
||||
plugin, found := generation.GetByChannelType(channelType.value)
|
||||
@@ -112,9 +138,7 @@ func TestBuiltInTaskPluginResponsesAndUsageContracts(t *testing.T) {
|
||||
assert.True(t, callable, hook)
|
||||
}
|
||||
for _, hook := range []string{"extractUsage", "extractUsageOnComplete"} {
|
||||
callable, callableErr := plugin.Engine.HasExport(t.Context(), hook)
|
||||
require.NoError(t, callableErr)
|
||||
assert.True(t, callable, hook)
|
||||
assert.True(t, plugin.Engine.HasExport(hook), hook)
|
||||
}
|
||||
require.NotEmpty(t, plugin.Meta.UsageSchema)
|
||||
for usageKey, schema := range plugin.Meta.UsageSchema {
|
||||
@@ -155,3 +179,236 @@ func TestBuiltInResponsesDecodersEchoChannelMappedAlias(t *testing.T) {
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestBuiltInPluginsAddressNewAPIUpstreamOnNativeRoutes(t *testing.T) {
|
||||
generation := jsplugin.DefaultRegistry.Generation()
|
||||
require.NotNil(t, generation)
|
||||
|
||||
const base = "https://upstream.example"
|
||||
cases := []struct {
|
||||
key string
|
||||
driver map[string]any
|
||||
vendorKey string
|
||||
vendorAuthPrefix string
|
||||
vendorSubmit string
|
||||
gatewaySubmit string
|
||||
vendorQuery string
|
||||
gatewayQuery string
|
||||
content string
|
||||
legacyKeyHeuristic bool
|
||||
}{
|
||||
{
|
||||
key: "doubao",
|
||||
driver: map[string]any{"action": "text_to_video", "model": "doubao-seedance-1-0-pro-250528", "upstreamModel": "doubao-seedance-1-0-pro-250528", "requestBody": map[string]any{"model": "doubao-seedance-1-0-pro-250528", "prompt": "a cat"}},
|
||||
vendorKey: "vendor-key",
|
||||
vendorAuthPrefix: "Bearer vendor-key",
|
||||
vendorSubmit: "/api/v3/contents/generations/tasks",
|
||||
gatewaySubmit: "/doubao/api/v3/contents/generations/tasks",
|
||||
vendorQuery: "/api/v3/contents/generations/tasks/tid",
|
||||
gatewayQuery: "/doubao/api/v3/contents/generations/tasks/tid",
|
||||
},
|
||||
{
|
||||
key: "alibaba",
|
||||
driver: map[string]any{"action": "text_to_video", "model": "wan2.2-t2v-plus", "upstreamModel": "wan2.2-t2v-plus", "requestBody": map[string]any{"model": "wan2.2-t2v-plus", "prompt": "a cat"}},
|
||||
vendorKey: "vendor-key",
|
||||
vendorAuthPrefix: "Bearer vendor-key",
|
||||
vendorSubmit: "/api/v1/services/aigc/video-generation/video-synthesis",
|
||||
gatewaySubmit: "/ali/api/v1/services/aigc/video-generation/video-synthesis",
|
||||
vendorQuery: "/api/v1/tasks/tid",
|
||||
gatewayQuery: "/ali/api/v1/tasks/tid",
|
||||
},
|
||||
{
|
||||
key: "kling",
|
||||
driver: map[string]any{"action": "text_to_video", "model": "kling-v1", "upstreamModel": "kling-v1", "requestBody": map[string]any{"prompt": "a cat"}},
|
||||
vendorKey: "ak|sk",
|
||||
vendorAuthPrefix: "Bearer ey",
|
||||
vendorSubmit: "/v1/videos/text2video",
|
||||
gatewaySubmit: "/kling/v1/videos/text2video",
|
||||
vendorQuery: "/v1/videos/text2video/tid",
|
||||
gatewayQuery: "/kling/v1/videos/text2video/tid",
|
||||
legacyKeyHeuristic: true,
|
||||
},
|
||||
{
|
||||
key: "jimeng",
|
||||
driver: map[string]any{"action": "text_to_video", "model": "jimeng_vgfm_t2v_l20", "upstreamModel": "jimeng_vgfm_t2v_l20", "requestBody": map[string]any{"prompt": "a cat"}},
|
||||
vendorKey: "ak|sk",
|
||||
vendorAuthPrefix: "HMAC-SHA256",
|
||||
vendorSubmit: "/?Action=CVSync2AsyncSubmitTask&Version=2022-08-31",
|
||||
gatewaySubmit: "/jimeng/?Action=CVSync2AsyncSubmitTask&Version=2022-08-31",
|
||||
vendorQuery: "/?Action=CVSync2AsyncGetResult&Version=2022-08-31",
|
||||
gatewayQuery: "/jimeng/?Action=CVSync2AsyncGetResult&Version=2022-08-31",
|
||||
legacyKeyHeuristic: true,
|
||||
},
|
||||
{
|
||||
key: "sunoapi",
|
||||
driver: map[string]any{"action": "MUSIC", "model": "suno_music", "upstreamModel": "suno_music", "requestBody": map[string]any{"prompt": "a song"}},
|
||||
vendorKey: "vendor-key",
|
||||
vendorAuthPrefix: "Bearer vendor-key",
|
||||
vendorSubmit: "/suno/submit/MUSIC",
|
||||
gatewaySubmit: "/suno/submit/MUSIC",
|
||||
vendorQuery: "/suno/fetch",
|
||||
gatewayQuery: "/suno/fetch",
|
||||
},
|
||||
{
|
||||
key: "sora",
|
||||
driver: map[string]any{"action": "text_to_video", "model": "sora-2", "upstreamModel": "sora-2", "requestBody": map[string]any{"prompt": "a cat"}},
|
||||
vendorKey: "vendor-key",
|
||||
vendorAuthPrefix: "Bearer vendor-key",
|
||||
vendorSubmit: "/v1/videos",
|
||||
gatewaySubmit: "/v1/videos",
|
||||
vendorQuery: "/v1/videos/tid",
|
||||
gatewayQuery: "/v1/videos/tid",
|
||||
content: "/v1/videos/tid/content",
|
||||
},
|
||||
{
|
||||
key: "xai",
|
||||
driver: map[string]any{"action": "text_to_video", "model": "grok-imagine-video", "upstreamModel": "grok-imagine-video", "requestBody": map[string]any{"model": "grok-imagine-video", "prompt": "a cat"}},
|
||||
vendorKey: "vendor-key",
|
||||
vendorAuthPrefix: "Bearer vendor-key",
|
||||
vendorSubmit: "/v1/videos/generations",
|
||||
gatewaySubmit: "/xai/v1/videos/generations",
|
||||
vendorQuery: "/v1/videos/tid",
|
||||
gatewayQuery: "/xai/v1/videos/tid",
|
||||
},
|
||||
{
|
||||
key: "hailuo",
|
||||
driver: map[string]any{"action": "text_to_video", "model": "MiniMax-Hailuo-2.3", "upstreamModel": "MiniMax-Hailuo-2.3", "requestBody": map[string]any{"model": "MiniMax-Hailuo-2.3", "prompt": "a cat"}},
|
||||
vendorKey: "vendor-key",
|
||||
vendorAuthPrefix: "Bearer vendor-key",
|
||||
vendorSubmit: "/v1/video_generation",
|
||||
gatewaySubmit: "/hailuo/v1/video_generation",
|
||||
vendorQuery: "/v1/query/video_generation?task_id=tid",
|
||||
gatewayQuery: "/hailuo/v1/query/video_generation/tid",
|
||||
},
|
||||
{
|
||||
key: "vidu",
|
||||
driver: map[string]any{"action": "text_to_video", "model": "viduq2", "upstreamModel": "viduq2", "requestBody": map[string]any{"model": "viduq2", "prompt": "a cat"}},
|
||||
vendorKey: "vendor-key",
|
||||
vendorAuthPrefix: "Token vendor-key",
|
||||
vendorSubmit: "/ent/v2/text2video",
|
||||
gatewaySubmit: "/vidu/ent/v2/text2video",
|
||||
vendorQuery: "/ent/v2/tasks/tid/creations",
|
||||
gatewayQuery: "/vidu/ent/v2/tasks/tid/creations",
|
||||
},
|
||||
}
|
||||
variants := []struct {
|
||||
name string
|
||||
upstream map[string]any
|
||||
gateway bool
|
||||
legacyKey bool
|
||||
}{
|
||||
{"vendor signal", map[string]any{"kind": "vendor"}, false, false},
|
||||
{"no signal from an older host", nil, false, false},
|
||||
{"new_api signal", map[string]any{"kind": "new_api"}, true, false},
|
||||
{"legacy sk- key without a signal", nil, true, true},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.key, func(t *testing.T) {
|
||||
plugin, found := generation.Get(tc.key)
|
||||
require.True(t, found)
|
||||
require.True(t, plugin.Meta.SupportsUpstream(jsplugin.UpstreamKindNewAPI), "%s must declare new_api to be bindable to a New API channel", tc.key)
|
||||
call := func(hook string, args ...any) map[string]any {
|
||||
value, err := plugin.Engine.Call(t.Context(), hook, args...)
|
||||
require.NoError(t, err, hook)
|
||||
encoded, err := common.Marshal(value)
|
||||
require.NoError(t, err)
|
||||
var descriptor map[string]any
|
||||
require.NoError(t, common.Unmarshal(encoded, &descriptor))
|
||||
return descriptor
|
||||
}
|
||||
for _, variant := range variants {
|
||||
if variant.legacyKey && !tc.legacyKeyHeuristic {
|
||||
continue
|
||||
}
|
||||
t.Run(variant.name, func(t *testing.T) {
|
||||
apiKey := tc.vendorKey
|
||||
if variant.legacyKey {
|
||||
apiKey = "sk-legacy"
|
||||
} else if variant.gateway {
|
||||
apiKey = "sk-gateway"
|
||||
}
|
||||
authHeader := apiKey
|
||||
if variant.gateway && !variant.legacyKey {
|
||||
authHeader = "Bearer " + apiKey
|
||||
}
|
||||
ctx := map[string]any{"baseUrl": base, "apiKey": apiKey, "authHeader": authHeader, "publicTaskId": "task_pub", "requestHeaders": map[string]any{}, "files": []any{}}
|
||||
for key, value := range tc.driver {
|
||||
ctx[key] = value
|
||||
}
|
||||
if variant.upstream != nil {
|
||||
ctx["upstream"] = variant.upstream
|
||||
}
|
||||
wantSubmit, wantQuery := tc.vendorSubmit, tc.vendorQuery
|
||||
if variant.gateway {
|
||||
wantSubmit, wantQuery = tc.gatewaySubmit, tc.gatewayQuery
|
||||
}
|
||||
|
||||
submit := call("buildSubmitRequest", ctx)
|
||||
assert.Equal(t, base+wantSubmit, submit["url"])
|
||||
headers, _ := submit["headers"].(map[string]any)
|
||||
authorization, _ := headers["Authorization"].(string)
|
||||
if variant.gateway {
|
||||
assert.Equal(t, "Bearer "+apiKey, authorization, "a gateway receives its own token as a Bearer header")
|
||||
} else {
|
||||
assert.True(t, strings.HasPrefix(authorization, tc.vendorAuthPrefix), "vendor Authorization %q", authorization)
|
||||
}
|
||||
|
||||
queryCtx := map[string]any{"taskId": "tid", "publicTaskId": "task_pub", "action": tc.driver["action"], "model": tc.driver["model"], "upstreamModel": tc.driver["upstreamModel"], "baseUrl": base, "apiKey": apiKey, "authHeader": authHeader, "data": nil, "state": nil}
|
||||
if variant.upstream != nil {
|
||||
queryCtx["upstream"] = variant.upstream
|
||||
}
|
||||
var query map[string]any
|
||||
if plugin.Meta.FetchMode == "batch" {
|
||||
query = call("buildBatchQueryRequest", queryCtx, []any{queryCtx})
|
||||
} else {
|
||||
query = call("buildQueryRequest", queryCtx)
|
||||
}
|
||||
assert.Equal(t, base+wantQuery, query["url"])
|
||||
|
||||
if tc.content == "" {
|
||||
return
|
||||
}
|
||||
contentCtx := map[string]any{"baseUrl": base, "apiKey": apiKey, "authHeader": authHeader, "artifactKey": "video", "upstreamTaskId": "tid", "data": map[string]any{}, "clientRequest": map[string]any{"method": "GET", "headers": map[string]any{}}}
|
||||
if variant.upstream != nil {
|
||||
contentCtx["upstream"] = variant.upstream
|
||||
}
|
||||
assert.Equal(t, base+tc.content, call("buildContentRequest", contentCtx)["url"])
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("alibaba asks only DashScope for an interleaved image stream", func(t *testing.T) {
|
||||
plugin, found := generation.Get("alibaba")
|
||||
require.True(t, found)
|
||||
for _, variant := range []struct {
|
||||
name string
|
||||
upstream map[string]any
|
||||
wantResponseType string
|
||||
wantSSEHeader bool
|
||||
}{
|
||||
{"vendor", map[string]any{"kind": "vendor"}, "sse", true},
|
||||
{"new_api", map[string]any{"kind": "new_api"}, "json", false},
|
||||
} {
|
||||
t.Run(variant.name, func(t *testing.T) {
|
||||
value, err := plugin.Engine.Call(t.Context(), "buildSubmitRequest", map[string]any{
|
||||
"baseUrl": base, "apiKey": "key", "authHeader": "key", "publicTaskId": "task_pub", "upstream": variant.upstream,
|
||||
"model": "wan2.6-image", "upstreamModel": "wan2.6-image",
|
||||
"requestBody": map[string]any{"model": "wan2.6-image", "prompt": "a cat", "metadata": map[string]any{"upstream_mode": "sync", "parameters": map[string]any{"enable_interleave": true}}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
encoded, err := common.Marshal(value)
|
||||
require.NoError(t, err)
|
||||
var descriptor struct {
|
||||
URL string `json:"url"`
|
||||
ResponseType string `json:"responseType"`
|
||||
Headers map[string]string `json:"headers"`
|
||||
}
|
||||
require.NoError(t, common.Unmarshal(encoded, &descriptor))
|
||||
assert.Equal(t, variant.wantResponseType, descriptor.ResponseType, "a gateway native route aggregates the vendor stream and answers with JSON")
|
||||
_, hasSSEHeader := descriptor.Headers["X-DashScope-Sse"]
|
||||
assert.Equal(t, variant.wantSSEHeader, hasSSEHeader)
|
||||
})
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,6 +1,32 @@
|
||||
package plugins_test
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/pkg/billingexpr"
|
||||
"github.com/QuantumNous/new-api/pkg/jsplugin"
|
||||
builtinplugins "github.com/QuantumNous/new-api/plugins"
|
||||
"github.com/QuantumNous/new-api/relay"
|
||||
taskplugin "github.com/QuantumNous/new-api/relay/channel/task/jsplugin"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
const (
|
||||
doubaoImageRoute = "/doubao/api/v3/images/generations"
|
||||
doubaoBaseURL = "https://ark.cn-beijing.volces.com"
|
||||
)
|
||||
|
||||
func TestDoubaoResponsesProtocol(t *testing.T) {
|
||||
testVideoResponsesProtocol(t, videoResponsesTestCase{
|
||||
@@ -29,3 +55,960 @@ func TestDoubaoResponsesProtocol(t *testing.T) {
|
||||
wantVendorName: "doubao",
|
||||
})
|
||||
}
|
||||
|
||||
func newDoubaoPlugin(t *testing.T) (*jsplugin.Registry, *jsplugin.LoadedPlugin) {
|
||||
t.Helper()
|
||||
source, err := builtinplugins.Source("doubao")
|
||||
require.NoError(t, err)
|
||||
registry := jsplugin.NewRegistry()
|
||||
plugin, err := registry.RegisterFactory(source, jsplugin.Options{Key: "doubao"})
|
||||
require.NoError(t, err)
|
||||
return registry, plugin
|
||||
}
|
||||
|
||||
func decodeDoubaoImage(t *testing.T, registry *jsplugin.Registry, plugin *jsplugin.LoadedPlugin, body map[string]any) (map[string]any, error) {
|
||||
t.Helper()
|
||||
binding, found := registry.Generation().LookupDeclaredRoute(http.MethodPost, doubaoImageRoute)
|
||||
require.True(t, found)
|
||||
require.Equal(t, jsplugin.RouteTypeSubmit, binding.Route.Type)
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "native", []string{binding.Route.Decode}, map[string]any{
|
||||
"path": doubaoImageRoute, "body": map[string]any{"kind": "json", "value": body},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return alibabaObject(t, value), nil
|
||||
}
|
||||
|
||||
// Runs the production host validation, body conversion and usage extraction
|
||||
// for one normalized image request without contacting Volcengine.
|
||||
func submitDoubaoImage(t *testing.T, plugin *jsplugin.LoadedPlugin, action string, request map[string]any) (map[string]any, map[string]any, string) {
|
||||
t.Helper()
|
||||
modelName := request["model"].(string)
|
||||
// RelayTask copies the decode action onto the relay info before the adaptor runs.
|
||||
info := &relaycommon.RelayInfo{
|
||||
ChannelMeta: &relaycommon.ChannelMeta{ChannelBaseUrl: doubaoBaseURL, UpstreamModelName: modelName},
|
||||
OriginModelName: modelName,
|
||||
TaskRelayInfo: &relaycommon.TaskRelayInfo{PublicTaskID: "task_public", Action: action},
|
||||
}
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(info)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, doubaoImageRoute, nil)
|
||||
c.Set("task_request", request)
|
||||
require.Nil(t, adaptor.ValidateRequestAndSetAction(c, info))
|
||||
reader, err := adaptor.BuildRequestBody(c, info)
|
||||
require.NoError(t, err)
|
||||
encoded, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
var body map[string]any
|
||||
require.NoError(t, common.Unmarshal(encoded, &body))
|
||||
facts, err := adaptor.ExtractUsageFactsValidated(c, info)
|
||||
require.NoError(t, err)
|
||||
url, err := adaptor.BuildRequestURL(info)
|
||||
require.NoError(t, err)
|
||||
return body, alibabaObject(t, facts), url
|
||||
}
|
||||
|
||||
func TestDoubaoImageSubmission(t *testing.T) {
|
||||
registry, plugin := newDoubaoPlugin(t)
|
||||
reference := "https://cdn.example/reference.png"
|
||||
// Every model reports the four facts released with the shared image schema,
|
||||
// including those its own profile omits, so expressions saved before the
|
||||
// per-model profiles keep their charge.
|
||||
noImages := map[string]any{"images_up_to_1_5k": float64(0), "images_above_1_5k": float64(0), "input_images": float64(0), "layer_decomposition": false}
|
||||
facts := func(overrides map[string]any) map[string]any {
|
||||
merged := map[string]any{}
|
||||
maps.Copy(merged, noImages)
|
||||
maps.Copy(merged, overrides)
|
||||
return merged
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
model string
|
||||
body map[string]any
|
||||
wantAction string
|
||||
wantBody map[string]any // upstream fields whose value differs from the request
|
||||
wantFacts map[string]any
|
||||
}{
|
||||
{"text to image with a preset size", "doubao-seedream-4-0-250828",
|
||||
map[string]any{"prompt": "a cat", "size": "2K", "watermark": false, "response_format": "url", "stream": false, "seed": 42, "guidance_scale": 2.5},
|
||||
"text_to_image", nil, facts(map[string]any{"images_above_1_5k": float64(1)})},
|
||||
{"1K reserves the lower pricing tier", "doubao-seedream-4-0-250828",
|
||||
map[string]any{"prompt": "a cat", "size": "1K", "optimize_prompt_options": nil},
|
||||
"text_to_image", nil, facts(map[string]any{"images_up_to_1_5k": float64(1)})},
|
||||
{"lowercase presets are normalized", "doubao-seedream-4-0-250828",
|
||||
map[string]any{"prompt": "a cat", "size": "2k"},
|
||||
"text_to_image", map[string]any{"size": "2K"}, facts(map[string]any{"images_above_1_5k": float64(1)})},
|
||||
{"pixel sizes are normalized to WxH", "doubao-seedream-4-5-251128",
|
||||
map[string]any{"prompt": "a banner", "size": "3750 * 1250"},
|
||||
"text_to_image", map[string]any{"size": "3750x1250"}, facts(map[string]any{"images_above_1_5k": float64(1)})},
|
||||
{"the 2.61 megapixel boundary belongs to the lower tier", "doubao-seedream-5-0-pro-260628",
|
||||
map[string]any{"prompt": "a poster", "size": "1500X1740"},
|
||||
"text_to_image", map[string]any{"size": "1500x1740"}, facts(map[string]any{"images_up_to_1_5k": float64(1)})},
|
||||
{"5.0 pro 1.5K with fast prompt optimization", "doubao-seedream-5-0-pro-260628",
|
||||
map[string]any{"prompt": "a poster", "size": "1.5K", "optimize_prompt_options": map[string]any{"mode": "fast"}, "output_format": "png"},
|
||||
"text_to_image", nil, facts(map[string]any{"images_up_to_1_5k": float64(1)})},
|
||||
{"group generation estimates max_images and reference images", "doubao-seedream-5-0-lite-260128",
|
||||
map[string]any{"prompt": "a brand kit", "image": reference, "size": "2K", "sequential_image_generation": "auto", "sequential_image_generation_options": map[string]any{"max_images": 4}, "output_format": "png", "tools": []any{map[string]any{"type": "web_search"}}},
|
||||
"image_to_image", nil, facts(map[string]any{"images_above_1_5k": float64(4), "input_images": float64(1)})},
|
||||
{"group generation is capped by reference images", "doubao-seedream-4-5-251128",
|
||||
map[string]any{"prompt": "four seasons", "image": []any{reference, "https://cdn.example/second.png"}, "sequential_image_generation": "auto"},
|
||||
"image_to_image", nil, facts(map[string]any{"images_above_1_5k": float64(13), "input_images": float64(2)})},
|
||||
{"base64 references count as input images", "doubao-seedream-5-0-pro-260628",
|
||||
map[string]any{"prompt": "a cat", "image": []any{reference, "data:image/png;base64,iVBORw0KGgo="}, "size": "1K"},
|
||||
"image_to_image", nil, facts(map[string]any{"images_up_to_1_5k": float64(1), "input_images": float64(2)})},
|
||||
{"transparent background on 5.0 pro", "doubao-seedream-5-0-pro-260628",
|
||||
map[string]any{"prompt": "cut out the cat", "image": reference, "background": "transparent", "output_format": "png", "size": "1K"},
|
||||
"image_to_image", nil, facts(map[string]any{"images_up_to_1_5k": float64(1), "input_images": float64(1)})},
|
||||
{"background is forwarded to other models for upstream to decide", "doubao-seedream-5-0-lite-260128",
|
||||
map[string]any{"prompt": "cut out the cat", "background": "transparent", "size": "2K"},
|
||||
"text_to_image", nil, facts(map[string]any{"images_above_1_5k": float64(1)})},
|
||||
{"layer decomposition with auto size reserves every output at the higher tier", "doubao-seedream-5-0-pro-260628",
|
||||
map[string]any{"image": reference, "layer_decomposition": true, "size": "auto"},
|
||||
"image_to_image", nil, facts(map[string]any{"images_above_1_5k": float64(17), "input_images": float64(1), "layer_decomposition": true})},
|
||||
{"layer decomposition at 1K reserves the lower tier", "doubao-seedream-5-0-pro-260628",
|
||||
map[string]any{"image": reference, "layer_decomposition": true, "size": "1k"},
|
||||
"image_to_image", map[string]any{"size": "1K"}, facts(map[string]any{"images_up_to_1_5k": float64(17), "input_images": float64(1), "layer_decomposition": true})},
|
||||
{"5.0 flash layer decomposition at 1.5K", "doubao-seedream-5-0-flash-260915",
|
||||
map[string]any{"image": reference, "layer_decomposition": true, "size": "1.5K", "output_format": "png"},
|
||||
"image_to_image", nil, facts(map[string]any{"images_up_to_1_5k": float64(17), "input_images": float64(1), "layer_decomposition": true})},
|
||||
{"endpoint ids use the permissive profile", "ep-20260918-seedream",
|
||||
map[string]any{"prompt": "a cat", "size": "4K", "sequential_image_generation": "auto"},
|
||||
"text_to_image", nil, facts(map[string]any{"images_above_1_5k": float64(15)})},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
request := map[string]any{"model": tc.model}
|
||||
maps.Copy(request, tc.body)
|
||||
resolved, err := decodeDoubaoImage(t, registry, plugin, request)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.model, resolved["model"])
|
||||
assert.Equal(t, tc.wantAction, resolved["action"])
|
||||
body, facts, url := submitDoubaoImage(t, plugin, tc.wantAction, resolved["requestBody"].(map[string]any))
|
||||
assert.Equal(t, doubaoBaseURL+"/api/v3/images/generations", url)
|
||||
assert.Equal(t, tc.model, body["model"])
|
||||
assert.NotContains(t, body, "stream")
|
||||
for key, expected := range alibabaObject(t, tc.body) {
|
||||
if key == "stream" {
|
||||
continue
|
||||
}
|
||||
if override, ok := tc.wantBody[key]; ok {
|
||||
expected = override
|
||||
}
|
||||
assert.Contains(t, body, key)
|
||||
assert.Equal(t, expected, body[key], key)
|
||||
}
|
||||
assert.Equal(t, tc.wantFacts, facts)
|
||||
})
|
||||
}
|
||||
|
||||
// Legacy per-call pricing multiplies the price by every ratio, so the
|
||||
// reservation ratio is only the total requested output count: one 1K image
|
||||
// with two reference images reserves price × 1, not price × 1 × 2.
|
||||
t.Run("legacy per-call pricing reserves only the total output count", func(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
model string
|
||||
body map[string]any
|
||||
want float64
|
||||
}{
|
||||
{"1K image with two reference images", "doubao-seedream-5-0-pro-260628", map[string]any{"prompt": "a cat", "image": []any{reference, "https://cdn.example/second.png"}, "size": "1K"}, 1},
|
||||
{"group generation", "doubao-seedream-5-0-lite-260128", map[string]any{"prompt": "a brand kit", "image": reference, "size": "2K", "sequential_image_generation": "auto", "sequential_image_generation_options": map[string]any{"max_images": 4}}, 4},
|
||||
{"layer decomposition", "doubao-seedream-5-0-pro-260628", map[string]any{"image": reference, "layer_decomposition": true, "size": "auto"}, 17},
|
||||
} {
|
||||
request := map[string]any{"model": tc.model}
|
||||
maps.Copy(request, tc.body)
|
||||
resolved, err := decodeDoubaoImage(t, registry, plugin, request)
|
||||
require.NoError(t, err, tc.name)
|
||||
ctx := map[string]any{"upstreamModel": tc.model, "model": tc.model, "action": resolved["action"], "requestBody": resolved["requestBody"]}
|
||||
ctx["usagePurpose"] = "billing_ratios"
|
||||
value, err := plugin.Engine.Call(t.Context(), "extractUsage", ctx)
|
||||
require.NoError(t, err, tc.name)
|
||||
assert.Equal(t, map[string]any{"image_count": tc.want}, alibabaObject(t, value), tc.name)
|
||||
ctx["usagePurpose"] = "facts"
|
||||
value, err = plugin.Engine.Call(t.Context(), "extractUsage", ctx)
|
||||
require.NoError(t, err, tc.name)
|
||||
facts := alibabaObject(t, value)
|
||||
assert.NotContains(t, facts, "image_count", tc.name)
|
||||
assert.Equal(t, tc.want, facts["images_up_to_1_5k"].(float64)+facts["images_above_1_5k"].(float64), tc.name)
|
||||
}
|
||||
})
|
||||
|
||||
// Each model's pricing lists only the tiers it can output and layer
|
||||
// decomposition only where it is supported.
|
||||
t.Run("image models serve Responses and OpenAI Images host protocols", func(t *testing.T) {
|
||||
layered := []string{"images_up_to_1_5k", "images_above_1_5k", "input_images", "layer_decomposition"}
|
||||
oneTier := []string{"images_above_1_5k", "input_images"}
|
||||
for name, usageKeys := range map[string][]string{
|
||||
"doubao-seedream-5-0-pro-260628": layered,
|
||||
"doubao-seedream-5-0-flash-260915": layered,
|
||||
"doubao-seedream-5-0-lite-260128": oneTier,
|
||||
"doubao-seedream-5-0-260128": oneTier,
|
||||
"doubao-seedream-4-5-251128": oneTier,
|
||||
"doubao-seedream-4-0-250828": {"images_up_to_1_5k", "images_above_1_5k", "input_images"},
|
||||
} {
|
||||
_, found := registry.Generation().LookupEndpoint(http.MethodPost, "/v1/responses", name)
|
||||
assert.True(t, found, name)
|
||||
for _, path := range []string{"/v1/images/generations", "/v1/images/edits"} {
|
||||
binding, found := registry.Generation().LookupEndpoint(http.MethodPost, path, name)
|
||||
require.True(t, found, name+" "+path)
|
||||
assert.Equal(t, "openai_image", binding.Protocol)
|
||||
}
|
||||
_, found = registry.Generation().LookupEndpoint(http.MethodPost, "/v1/videos", name)
|
||||
assert.False(t, found, name)
|
||||
schema, examples := plugin.Meta.UsageForModel(name)
|
||||
assert.ElementsMatch(t, usageKeys, keysOf(schema), name)
|
||||
assert.NotEmpty(t, examples, name)
|
||||
}
|
||||
_, found := registry.Generation().LookupEndpoint(http.MethodPost, "/v1/videos", "doubao-seedance-2-0-260128")
|
||||
assert.True(t, found)
|
||||
_, found = registry.Generation().LookupEndpoint(http.MethodPost, "/v1/images/generations", "doubao-seedance-2-0-260128")
|
||||
assert.False(t, found)
|
||||
schema, _ := plugin.Meta.UsageForModel("doubao-seedance-2-0-260128")
|
||||
assert.ElementsMatch(t, []string{"tokens", "resolution", "video_input"}, keysOf(schema))
|
||||
schema, examples := plugin.Meta.UsageForModel("doubao-seedance-1-5-pro-251215")
|
||||
assert.ElementsMatch(t, []string{"tokens", "resolution", "generate_audio"}, keysOf(schema))
|
||||
assert.Equal(t, "boolean", schema["generate_audio"].Type)
|
||||
assert.NotEmpty(t, examples)
|
||||
})
|
||||
}
|
||||
|
||||
// Capability profiles follow the Ark model list: pricing lists only the
|
||||
// resolutions each Seedance model offers, reference video input only on
|
||||
// Seedance 2.x, and audio output only on Seedance 1.5 pro.
|
||||
func TestDoubaoSeedanceUsageFacts(t *testing.T) {
|
||||
_, plugin := newDoubaoPlugin(t)
|
||||
const (
|
||||
pro10 = "doubao-seedance-1-0-pro-250528"
|
||||
pro15 = "doubao-seedance-1-5-pro-251215"
|
||||
v20 = "doubao-seedance-2-0-260128"
|
||||
fast20 = "doubao-seedance-2-0-fast-260128"
|
||||
mini20 = "doubao-seedance-2-0-mini-260615"
|
||||
v25 = "doubao-seedance-2-5-260628"
|
||||
)
|
||||
families := []struct {
|
||||
models []string
|
||||
resolutions []string
|
||||
fields []string
|
||||
}{
|
||||
{[]string{pro10, "doubao-seedance-1-0-lite-t2v", "doubao-seedance-1-0-lite-i2v"}, []string{"480p", "720p", "1080p"}, []string{"resolution", "tokens"}},
|
||||
{[]string{pro15}, []string{"480p", "720p", "1080p"}, []string{"generate_audio", "resolution", "tokens"}},
|
||||
{[]string{v20}, []string{"480p", "720p", "1080p", "4k"}, []string{"resolution", "tokens", "video_input"}},
|
||||
{[]string{fast20, mini20}, []string{"480p", "720p"}, []string{"resolution", "tokens", "video_input"}},
|
||||
{[]string{v25}, []string{"480p", "720p", "1080p"}, []string{"resolution", "tokens", "video_input"}},
|
||||
}
|
||||
profiled := make([]string, 0, len(plugin.Meta.Models))
|
||||
for _, family := range families {
|
||||
for _, name := range family.models {
|
||||
profiled = append(profiled, name)
|
||||
t.Run(name, func(t *testing.T) {
|
||||
schema, examples := plugin.Meta.UsageForModel(name)
|
||||
assert.Equal(t, family.fields, slices.Sorted(maps.Keys(schema)))
|
||||
assert.Equal(t, family.resolutions, schema["resolution"].Enum)
|
||||
require.NotEmpty(t, examples)
|
||||
for _, example := range examples {
|
||||
assert.Equal(t, family.fields, slices.Sorted(maps.Keys(example.Facts)), example.Label)
|
||||
assert.Contains(t, family.resolutions, example.Facts["resolution"], example.Label)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
videoModels := make([]string, 0, len(profiled))
|
||||
for _, name := range plugin.Meta.Models {
|
||||
if strings.Contains(name, "seedance") {
|
||||
videoModels = append(videoModels, name)
|
||||
}
|
||||
}
|
||||
assert.ElementsMatch(t, videoModels, profiled, "every Seedance model selects a capability profile")
|
||||
|
||||
// A table price saved when every Seedance model declared video_input: its
|
||||
// last row is the unconditioned fallback.
|
||||
const sharedMatrix = `u("resolution") == "480p" && u("video_input") == "none" ? tier("480p·none", u("tokens") * 2 / 1000000) : ` +
|
||||
`u("resolution") == "480p" && u("video_input") == "video" ? tier("480p·video", u("tokens") * 3 / 1000000) : ` +
|
||||
`u("resolution") == "720p" && u("video_input") == "none" ? tier("720p·none", u("tokens") * 4 / 1000000) : ` +
|
||||
`u("resolution") == "720p" && u("video_input") == "video" ? tier("720p·video", u("tokens") * 5 / 1000000) : ` +
|
||||
`u("resolution") == "1080p" && u("video_input") == "none" ? tier("1080p·none", u("tokens") * 6 / 1000000) : ` +
|
||||
`u("resolution") == "1080p" && u("video_input") == "video" ? tier("1080p·video", u("tokens") * 7 / 1000000) : ` +
|
||||
`u("resolution") == "4k" && u("video_input") == "none" ? tier("4k·none", u("tokens") * 8 / 1000000) : ` +
|
||||
`tier("4k·video", u("tokens") * 9 / 1000000)`
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
model string
|
||||
request map[string]any
|
||||
want map[string]any
|
||||
wantErr string
|
||||
}{
|
||||
{"1.5 pro defaults to audio", pro15, map[string]any{"metadata": map[string]any{"resolution": "720p"}},
|
||||
map[string]any{"tokens": float64(108000), "resolution": "720p", "generate_audio": true, "video_input": "none"}, ""},
|
||||
{"1.5 pro silent output", pro15, map[string]any{"metadata": map[string]any{"resolution": "720p", "generate_audio": false}},
|
||||
map[string]any{"tokens": float64(108000), "resolution": "720p", "generate_audio": false, "video_input": "none"}, ""},
|
||||
{"2.0 has no audio fact", v20, map[string]any{"metadata": map[string]any{"resolution": "720p", "generate_audio": false}},
|
||||
map[string]any{"tokens": float64(108000), "resolution": "720p", "video_input": "none"}, ""},
|
||||
{"2.0 offers 4k", v20, map[string]any{"metadata": map[string]any{"resolution": "4k"}},
|
||||
map[string]any{"tokens": float64(972000), "resolution": "4k", "video_input": "none"}, ""},
|
||||
{"1.0 pro reports no reference video although its profile omits it", pro10, map[string]any{"metadata": map[string]any{"resolution": "1080p"}},
|
||||
map[string]any{"tokens": float64(243000), "resolution": "1080p", "video_input": "none"}, ""},
|
||||
{"mini reserves its highest tier when the resolution is left to Ark", mini20, map[string]any{},
|
||||
map[string]any{"tokens": float64(108000), "resolution": "720p", "video_input": "none"}, ""},
|
||||
{"mini rejects 1080p", mini20, map[string]any{"metadata": map[string]any{"resolution": "1080p"}}, nil, mini20 + " resolution must be one of 480p, 720p"},
|
||||
{"mini rejects a 1080p size", mini20, map[string]any{"size": "1920x1080"}, nil, mini20 + " resolution must be one of 480p, 720p"},
|
||||
{"fast rejects 4k", fast20, map[string]any{"metadata": map[string]any{"resolution": "4k"}}, nil, fast20 + " resolution must be one of 480p, 720p"},
|
||||
{"2.5 rejects 4k", v25, map[string]any{"metadata": map[string]any{"resolution": "4K"}}, nil, v25 + " resolution must be one of 480p, 720p, 1080p"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
request := map[string]any{"model": tc.model, "prompt": "a cat", "seconds": float64(5)}
|
||||
maps.Copy(request, tc.request)
|
||||
info := &relaycommon.RelayInfo{
|
||||
ChannelMeta: &relaycommon.ChannelMeta{ChannelBaseUrl: doubaoBaseURL, UpstreamModelName: tc.model},
|
||||
OriginModelName: tc.model,
|
||||
TaskRelayInfo: &relaycommon.TaskRelayInfo{PublicTaskID: "task_public", Action: "text_to_video"},
|
||||
}
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(info)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/videos", nil)
|
||||
c.Set("task_request", request)
|
||||
taskErr := adaptor.ValidateRequestAndSetAction(c, info)
|
||||
if tc.wantErr != "" {
|
||||
require.NotNil(t, taskErr)
|
||||
assert.Equal(t, http.StatusBadRequest, taskErr.StatusCode)
|
||||
assert.Contains(t, taskErr.Message, tc.wantErr)
|
||||
return
|
||||
}
|
||||
require.Nil(t, taskErr)
|
||||
reader, err := adaptor.BuildRequestBody(c, info)
|
||||
require.NoError(t, err)
|
||||
encoded, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
var body map[string]any
|
||||
require.NoError(t, common.Unmarshal(encoded, &body))
|
||||
metadata, _ := tc.request["metadata"].(map[string]any)
|
||||
assert.Equal(t, metadata["generate_audio"], body["generate_audio"])
|
||||
facts, err := adaptor.ExtractUsageFactsValidated(c, info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.want, alibabaObject(t, facts))
|
||||
_, trace, err := billingexpr.RunExprWithRequest(sharedMatrix, billingexpr.TokenParams{}, billingexpr.RequestInput{Usage: facts})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.want["resolution"].(string)+"·none", trace.MatchedTier, "saved table prices keep the no-video tier")
|
||||
})
|
||||
}
|
||||
|
||||
// Ark reads duration -1 as "the model picks the length" (Seedance 1.5 pro
|
||||
// and 2.x; 2.5 also defaults to it). The reservation covers the longest
|
||||
// output: 720p is 1280 × 720 × 24 / 1024 = 21,600 tokens per second.
|
||||
t.Run("duration -1 reaches Ark and reserves the longest duration", func(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
model string
|
||||
request any
|
||||
wantDuration any
|
||||
wantTokens float64
|
||||
}{
|
||||
{"2.5 metadata on /v1/video/generations", v25, relaycommon.TaskSubmitReq{
|
||||
Model: v25, Prompt: "a cat", Metadata: map[string]any{"duration": -1, "resolution": "720p"},
|
||||
}, float64(-1), 30 * 21600},
|
||||
{"2.5 without a duration keeps the Ark default", v25, map[string]any{
|
||||
"model": v25, "prompt": "a cat", "metadata": map[string]any{"resolution": "720p"},
|
||||
}, nil, 30 * 21600},
|
||||
{"2.0 seconds from Responses or /v1/videos", v20, map[string]any{
|
||||
"model": v20, "prompt": "a cat", "seconds": -1, "metadata": map[string]any{"resolution": "720p"},
|
||||
}, float64(-1), 15 * 21600},
|
||||
{"1.5 pro", pro15, map[string]any{
|
||||
"model": pro15, "prompt": "a cat", "metadata": map[string]any{"duration": -1, "resolution": "720p"},
|
||||
}, float64(-1), 12 * 21600},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
info := &relaycommon.RelayInfo{
|
||||
ChannelMeta: &relaycommon.ChannelMeta{ChannelBaseUrl: doubaoBaseURL, UpstreamModelName: tc.model},
|
||||
OriginModelName: tc.model,
|
||||
TaskRelayInfo: &relaycommon.TaskRelayInfo{PublicTaskID: "task_public", Action: "text_to_video"},
|
||||
}
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(info)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/video/generations", nil)
|
||||
c.Set("task_request", tc.request)
|
||||
require.Nil(t, adaptor.ValidateRequestAndSetAction(c, info))
|
||||
reader, err := adaptor.BuildRequestBody(c, info)
|
||||
require.NoError(t, err)
|
||||
encoded, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
var body map[string]any
|
||||
require.NoError(t, common.Unmarshal(encoded, &body))
|
||||
assert.Equal(t, tc.wantDuration, body["duration"])
|
||||
facts, err := adaptor.ExtractUsageFactsValidated(c, info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantTokens, facts["tokens"])
|
||||
})
|
||||
}
|
||||
|
||||
decode := func(body map[string]any) (any, error) {
|
||||
return plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_video", "decodeRequest"}, map[string]any{
|
||||
"model": v20, "body": map[string]any{"kind": "json", "value": body},
|
||||
})
|
||||
}
|
||||
value, err := decode(map[string]any{"model": v20, "prompt": "a cat", "duration": -1})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, float64(-1), alibabaObject(t, value)["requestBody"].(map[string]any)["seconds"])
|
||||
_, err = decode(map[string]any{"model": v20, "prompt": "a cat", "seconds": -2})
|
||||
require.ErrorContains(t, err, "seconds must be -1 or between 1 and 3600")
|
||||
})
|
||||
|
||||
t.Run("completion overlays only resolutions the model offers", func(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
model string
|
||||
resolution string
|
||||
want map[string]any
|
||||
}{
|
||||
{mini20, "1080p", map[string]any{"tokens": float64(90000)}},
|
||||
{v25, "4k", map[string]any{"tokens": float64(90000)}},
|
||||
{v20, "4k", map[string]any{"tokens": float64(90000), "resolution": "4k"}},
|
||||
} {
|
||||
queryContext := map[string]any{"model": tc.model, "upstreamModel": tc.model, "action": "text_to_video"}
|
||||
body := map[string]any{
|
||||
"status": "succeeded",
|
||||
"usage": map[string]any{"completion_tokens": 90000},
|
||||
"content": map[string]any{"video_url": "https://cdn.example/video.mp4", "resolution": tc.resolution},
|
||||
}
|
||||
value, err := plugin.Engine.Call(t.Context(), "extractUsageOnComplete", queryContext, map[string]any{"status": "SUCCESS"}, body)
|
||||
require.NoError(t, err, tc.model)
|
||||
assert.Equal(t, tc.want, alibabaObject(t, value), tc.model)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func keysOf(schema map[string]jsplugin.UsageFieldSchema) []string {
|
||||
keys := make([]string, 0, len(schema))
|
||||
for key := range schema {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
return keys
|
||||
}
|
||||
|
||||
// Rejections mirror the Ark API reference; the model-capability cases were
|
||||
// confirmed against live 400 responses (2026-09).
|
||||
func TestDoubaoImageValidation(t *testing.T) {
|
||||
registry, plugin := newDoubaoPlugin(t)
|
||||
const pro, flash, lite, v45, v40 = "doubao-seedream-5-0-pro-260628", "doubao-seedream-5-0-flash-260915", "doubao-seedream-5-0-lite-260128", "doubao-seedream-4-5-251128", "doubao-seedream-4-0-250828"
|
||||
reference := "https://cdn.example/1.png"
|
||||
manyReferences := make([]any, 0, 11)
|
||||
for index := range 11 {
|
||||
manyReferences = append(manyReferences, fmt.Sprintf("https://cdn.example/%d.png", index))
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
model string
|
||||
body map[string]any
|
||||
wantErr string
|
||||
}{
|
||||
{"prompt is required", v40, map[string]any{}, "prompt is required"},
|
||||
{"client streaming is rejected", v40, map[string]any{"prompt": "a cat", "stream": true}, "stream is not supported"},
|
||||
{"base64 responses are not deliverable", v40, map[string]any{"prompt": "a cat", "response_format": "b64_json"}, "response_format must be url"},
|
||||
{"5.0 pro rejects group generation", pro, map[string]any{"prompt": "a cat", "sequential_image_generation": "auto"}, "sequential_image_generation is not supported"},
|
||||
{"5.0 pro rejects sequential_image_generation even when disabled", pro, map[string]any{"prompt": "a cat", "sequential_image_generation": "disabled"}, "sequential_image_generation is not supported"},
|
||||
{"5.0 flash rejects group generation", flash, map[string]any{"prompt": "a cat", "sequential_image_generation": "auto"}, "sequential_image_generation is not supported"},
|
||||
{"sequential_image_generation is an enum", lite, map[string]any{"prompt": "a cat", "sequential_image_generation": "on"}, "must be auto or disabled"},
|
||||
{"sequential options must be an object", lite, map[string]any{"prompt": "a cat", "sequential_image_generation": "auto", "sequential_image_generation_options": 3}, "sequential_image_generation_options must be an object"},
|
||||
{"max_images is bounded", lite, map[string]any{"prompt": "a cat", "sequential_image_generation": "auto", "sequential_image_generation_options": map[string]any{"max_images": 16}}, "max_images must be an integer between 1 and 15"},
|
||||
{"reference images are bounded", pro, map[string]any{"prompt": "a cat", "image": manyReferences}, "at most 10 reference images"},
|
||||
{"fast prompt optimization is unsupported on 5.0 flash", flash, map[string]any{"prompt": "a cat", "optimize_prompt_options": map[string]any{"mode": "fast"}}, "mode fast is not supported"},
|
||||
{"fast prompt optimization is unsupported on 5.0 lite", lite, map[string]any{"prompt": "a cat", "optimize_prompt_options": map[string]any{"mode": "fast"}}, "mode fast is not supported"},
|
||||
{"fast prompt optimization is unsupported on 4.5", v45, map[string]any{"prompt": "a cat", "optimize_prompt_options": map[string]any{"mode": "fast"}}, "mode fast is not supported"},
|
||||
{"prompt optimization mode is an enum", v40, map[string]any{"prompt": "a cat", "optimize_prompt_options": map[string]any{"mode": "turbo"}}, "must be standard or fast"},
|
||||
{"prompt optimization options must be an object", v40, map[string]any{"prompt": "a cat", "optimize_prompt_options": 123}, "optimize_prompt_options must be an object"},
|
||||
{"output_format is an enum", pro, map[string]any{"prompt": "a cat", "output_format": "webp"}, "output_format must be png or jpeg"},
|
||||
{"output_format follows the model", v40, map[string]any{"prompt": "a cat", "output_format": "png"}, "output_format is not supported"},
|
||||
{"background is a string enum", pro, map[string]any{"prompt": "a cat", "image": reference, "background": true}, "background must be opaque or transparent"},
|
||||
{"transparent background needs exactly one input image", pro, map[string]any{"prompt": "a cat", "background": "transparent"}, "requires exactly one input image"},
|
||||
{"transparent background rejects jpeg output", pro, map[string]any{"prompt": "a cat", "image": reference, "background": "transparent", "output_format": "jpeg"}, "cannot be combined with output_format jpeg"},
|
||||
{"tools must be objects with a type", lite, map[string]any{"prompt": "a cat", "tools": []any{map[string]any{}}}, "tools must be an array of objects with a string type"},
|
||||
{"tools must be an array", lite, map[string]any{"prompt": "a cat", "tools": map[string]any{"type": "web_search"}}, "tools must be an array of objects with a string type"},
|
||||
{"tools follow the model", pro, map[string]any{"prompt": "a cat", "tools": []any{map[string]any{"type": "web_search"}}}, "tools are not supported"},
|
||||
{"seed must be an integer", v40, map[string]any{"prompt": "a cat", "seed": "abc"}, "seed must be an integer"},
|
||||
{"seed rejects fractions", v40, map[string]any{"prompt": "a cat", "seed": 1.5}, "seed must be an integer"},
|
||||
{"guidance_scale must be a number", v40, map[string]any{"prompt": "a cat", "guidance_scale": "x"}, "guidance_scale must be a number"},
|
||||
{"size must be a string", v40, map[string]any{"prompt": "a cat", "size": 2048}, "size must be a string"},
|
||||
{"presets follow the model", v45, map[string]any{"prompt": "a cat", "size": "1K"}, "size must be one of 2K, 4K"},
|
||||
{"5.0 flash tops out at 2K", flash, map[string]any{"prompt": "a cat", "size": "4K"}, "size must be one of 1K, 1.5K, 2K"},
|
||||
{"pixel sizes follow the model", v45, map[string]any{"prompt": "a cat", "size": "1500x1500"}, "outside the model's pixel"},
|
||||
{"aspect ratios are bounded", v40, map[string]any{"prompt": "a cat", "size": "8192x480"}, "outside the model's pixel"},
|
||||
{"layer decomposition needs one image", pro, map[string]any{"layer_decomposition": true}, "requires exactly one input image"},
|
||||
{"layer decomposition follows the model", v40, map[string]any{"prompt": "a cat", "image": reference, "layer_decomposition": true}, "layer_decomposition is not supported"},
|
||||
{"layer decomposition sizes are presets", pro, map[string]any{"image": reference, "layer_decomposition": true, "size": "2048x2048"}, "layer decomposition sizes must be one of"},
|
||||
{"auto size needs layer decomposition", v40, map[string]any{"prompt": "a cat", "size": "auto"}, "size auto is only supported"},
|
||||
{"image references must be URLs", v40, map[string]any{"prompt": "a cat", "image": "ftp://cdn.example/1.png"}, "image must be an HTTP URL"},
|
||||
{"base64 references need the full data URI", v40, map[string]any{"prompt": "a cat", "image": "data:image/png,iVBORw0KGgo="}, "image must be an HTTP URL"},
|
||||
{"base64 formats are lowercase", v40, map[string]any{"prompt": "a cat", "image": "data:image/PNG;base64,iVBORw0KGgo="}, "image must be an HTTP URL"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
request := map[string]any{"model": tc.model}
|
||||
maps.Copy(request, tc.body)
|
||||
_, err := decodeDoubaoImage(t, registry, plugin, request)
|
||||
require.ErrorContains(t, err, tc.wantErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Fixtures mirror real Ark responses captured on 2026-09-18 (signed URLs replaced).
|
||||
func TestDoubaoImageResults(t *testing.T) {
|
||||
_, plugin := newDoubaoPlugin(t)
|
||||
const lite, pro = "doubao-seedream-5-0-lite-260128", "doubao-seedream-5-0-pro-260628"
|
||||
first, second := "https://ark-content.example/1.jpeg?sig=secret", "https://ark-content.example/2.jpeg?sig=secret"
|
||||
// 5.0 lite group of two plus one failed member; Ark echoes the canonical model name.
|
||||
groupBody := map[string]any{
|
||||
"model": "doubao-seedream-5-0-260128",
|
||||
"created": 1789733151,
|
||||
"data": []any{
|
||||
map[string]any{"url": first, "size": "2848x1600"},
|
||||
map[string]any{"url": second, "size": "2848x1600"},
|
||||
map[string]any{"error": map[string]any{"code": "OutputImageSensitiveContentDetected", "message": "blocked"}},
|
||||
},
|
||||
"usage": map[string]any{"generated_images": 2, "output_tokens": 35600, "total_tokens": 35600},
|
||||
}
|
||||
// 5.0 pro layer decomposition with size auto: base and first layer above 2.61 MP, second layer below.
|
||||
layerBody := map[string]any{
|
||||
"model": pro,
|
||||
"created": 1789733460,
|
||||
"data": []any{
|
||||
map[string]any{"url": "https://ark-content.example/base.jpeg?sig=secret", "size": "2848x1600", "output_format": "jpeg", "z_index": 0},
|
||||
map[string]any{"url": "https://ark-content.example/layer1.png?sig=secret", "size": "2848x1600", "output_format": "png", "z_index": 1, "name": "背景场景", "description": "室内木桌背景",
|
||||
"bounding_box": map[string]any{"absolute": []any{0, 0, 2848, 1600}, "normalized": []any{0, 0, 1000, 999}}},
|
||||
map[string]any{"url": "https://ark-content.example/layer2.png?sig=secret", "size": "1125x956", "output_format": "png", "z_index": 2, "name": "主体橘猫", "description": "坐姿橘猫",
|
||||
"bounding_box": map[string]any{"absolute": []any{955, 622, 2080, 1578}, "normalized": []any{335, 389, 730, 986}}},
|
||||
},
|
||||
"usage": map[string]any{"input_images": 1, "generated_images": 3, "output_tokens": 42316, "total_tokens": 42316},
|
||||
}
|
||||
// 5.0 pro single 1K image from two reference images.
|
||||
referenceBody := map[string]any{
|
||||
"model": pro,
|
||||
"created": 1789733390,
|
||||
"data": []any{map[string]any{"url": first, "size": "1424x800", "output_format": "jpeg"}},
|
||||
"usage": map[string]any{"input_images": 2, "generated_images": 1, "output_tokens": 4450, "total_tokens": 4450},
|
||||
}
|
||||
|
||||
parseResponse := func(t *testing.T, model string, request map[string]any, payload map[string]any) *relaycommon.TaskInfo {
|
||||
t.Helper()
|
||||
info := &relaycommon.RelayInfo{OriginModelName: model, ChannelMeta: &relaycommon.ChannelMeta{UpstreamModelName: model, ChannelBaseUrl: doubaoBaseURL}, TaskRelayInfo: &relaycommon.TaskRelayInfo{PublicTaskID: "task_public"}}
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(info)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, doubaoImageRoute, nil)
|
||||
c.Set("task_request", request)
|
||||
encoded, err := common.Marshal(payload)
|
||||
require.NoError(t, err)
|
||||
parsed, taskErr := adaptor.ParseResponse(c, &http.Response{StatusCode: http.StatusOK, Header: http.Header{"Content-Type": {"application/json"}}, Body: io.NopCloser(bytes.NewReader(encoded))}, info)
|
||||
require.Nil(t, taskErr)
|
||||
require.NotNil(t, parsed.Immediate)
|
||||
assert.Equal(t, "task_public", parsed.UpstreamTaskID)
|
||||
assert.JSONEq(t, string(encoded), string(parsed.TaskData))
|
||||
return parsed.Immediate
|
||||
}
|
||||
groupRequest := map[string]any{"model": lite, "prompt": "a cat", "sequential_image_generation": "auto"}
|
||||
queryContext := map[string]any{"upstreamModel": lite, "model": lite, "action": "text_to_image"}
|
||||
proQuery := map[string]any{"upstreamModel": pro, "model": pro, "action": "text_to_image"}
|
||||
|
||||
t.Run("a single-tier group settles every delivered image", func(t *testing.T) {
|
||||
immediate := parseResponse(t, lite, groupRequest, groupBody)
|
||||
assert.Equal(t, "SUCCESS", immediate.Status)
|
||||
assert.Equal(t, "100%", immediate.Progress)
|
||||
assert.Equal(t, first, immediate.Url)
|
||||
assert.Equal(t, map[string]any{"images_above_1_5k": float64(2)}, immediate.UsageFacts)
|
||||
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "native", []string{"imageCreated"}, map[string]any{}, map[string]any{"task_id": "task_public", "status": "SUCCESS", "data": groupBody})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, alibabaObject(t, groupBody), alibabaObject(t, value))
|
||||
|
||||
value, err = plugin.Engine.Call(t.Context(), "extractUsageOnSubmit", queryContext, groupBody)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"image_count": float64(2)}, alibabaObject(t, value), "legacy per-call pricing settles on the delivered count")
|
||||
})
|
||||
|
||||
// Legacy per-call pricing multiplies the price by every ratio the plugin
|
||||
// returns, so the submit-time settlement carries only the total delivered
|
||||
// output count; a mixed-tier result with a reference image must settle 5
|
||||
// images, not price × 4 × 1 × 1.
|
||||
t.Run("legacy per-call pricing settles the total output count", func(t *testing.T) {
|
||||
tieredBody := map[string]any{
|
||||
"model": pro,
|
||||
"data": []any{
|
||||
map[string]any{"url": first, "size": "1024x1024"},
|
||||
map[string]any{"url": second, "size": "1024x1024"},
|
||||
map[string]any{"url": first, "size": "1024x1024"},
|
||||
map[string]any{"url": second, "size": "1024x1024"},
|
||||
map[string]any{"url": first, "size": "2848x1600"},
|
||||
},
|
||||
"usage": map[string]any{"input_images": 1, "generated_images": 5},
|
||||
}
|
||||
value, err := plugin.Engine.Call(t.Context(), "extractUsageOnComplete", proQuery, map[string]any{"status": "SUCCESS"}, tieredBody)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"images_up_to_1_5k": float64(4), "images_above_1_5k": float64(1), "input_images": float64(1)}, alibabaObject(t, value), "task expressions keep the tiered facts")
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body map[string]any
|
||||
want float64
|
||||
}{
|
||||
{"mixed tiers with a reference image", tieredBody, 5},
|
||||
{"layer decomposition", layerBody, 3},
|
||||
{"single image from two reference images", referenceBody, 1},
|
||||
} {
|
||||
value, err := plugin.Engine.Call(t.Context(), "extractUsageOnSubmit", queryContext, tc.body)
|
||||
require.NoError(t, err, tc.name)
|
||||
assert.Equal(t, map[string]any{"image_count": tc.want}, alibabaObject(t, value), tc.name)
|
||||
}
|
||||
value, err = plugin.Engine.Call(t.Context(), "extractUsageOnSubmit", queryContext, map[string]any{"data": []any{map[string]any{"error": map[string]any{"code": "x"}}}})
|
||||
require.NoError(t, err)
|
||||
assert.Empty(t, alibabaObject(t, value), "no delivered image leaves the reservation in place")
|
||||
})
|
||||
|
||||
t.Run("mixed-tier layers settle per layer with the input image count", func(t *testing.T) {
|
||||
request := map[string]any{"model": pro, "image": "https://cdn.example/photo.png", "layer_decomposition": true, "size": "auto"}
|
||||
immediate := parseResponse(t, pro, request, layerBody)
|
||||
assert.Equal(t, "SUCCESS", immediate.Status)
|
||||
assert.Equal(t, map[string]any{"images_up_to_1_5k": float64(1), "images_above_1_5k": float64(2), "input_images": float64(1)}, immediate.UsageFacts)
|
||||
})
|
||||
|
||||
t.Run("reference images settle from usage.input_images", func(t *testing.T) {
|
||||
request := map[string]any{"model": pro, "prompt": "a cat", "image": []any{"https://cdn.example/a.png", "https://cdn.example/b.png", "https://cdn.example/c.png"}, "size": "1K"}
|
||||
immediate := parseResponse(t, pro, request, referenceBody)
|
||||
assert.Equal(t, map[string]any{"images_up_to_1_5k": float64(1), "images_above_1_5k": float64(0), "input_images": float64(2)}, immediate.UsageFacts)
|
||||
})
|
||||
|
||||
t.Run("invalid completion counts retain the reservation", func(t *testing.T) {
|
||||
for _, count := range []any{-1, 0, 1.5, 99, "2", 3} {
|
||||
payload := map[string]any{"data": []any{map[string]any{"url": first, "size": "2048x2048"}}, "usage": map[string]any{"generated_images": count}}
|
||||
immediate := parseResponse(t, lite, groupRequest, payload)
|
||||
assert.Equal(t, "SUCCESS", immediate.Status)
|
||||
assert.Nil(t, immediate.UsageFacts, "count %v must not replace the reservation", count)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("invalid input image counts keep the estimate while tiers settle", func(t *testing.T) {
|
||||
for _, count := range []any{-1, 1.5, 15, "2"} {
|
||||
payload := map[string]any{"data": []any{map[string]any{"url": first, "size": "1424x800"}}, "usage": map[string]any{"generated_images": 1, "input_images": count}}
|
||||
value, err := plugin.Engine.Call(t.Context(), "extractUsageOnComplete", proQuery, map[string]any{"status": "SUCCESS"}, payload)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"images_up_to_1_5k": float64(1), "images_above_1_5k": float64(0)}, alibabaObject(t, value), "input_images %v", count)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("missing sizes keep the estimated tiers", func(t *testing.T) {
|
||||
payload := map[string]any{"data": []any{map[string]any{"url": first, "size": "2048x2048"}, map[string]any{"url": second}}, "usage": map[string]any{"generated_images": 2, "input_images": 1}}
|
||||
value, err := plugin.Engine.Call(t.Context(), "extractUsageOnComplete", proQuery, map[string]any{"status": "SUCCESS"}, payload)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"input_images": float64(1)}, alibabaObject(t, value))
|
||||
// A single-tier model needs no size to know the tier.
|
||||
value, err = plugin.Engine.Call(t.Context(), "extractUsageOnComplete", queryContext, map[string]any{"status": "SUCCESS"}, payload)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"images_above_1_5k": float64(2), "input_images": float64(1)}, alibabaObject(t, value))
|
||||
})
|
||||
|
||||
t.Run("missing usage settles from image payload sizes", func(t *testing.T) {
|
||||
payload := map[string]any{"data": []any{map[string]any{"url": first, "size": "2848x1600"}, map[string]any{"url": second, "size": "1024x1024"}, map[string]any{"error": map[string]any{"code": "x"}}}}
|
||||
value, err := plugin.Engine.Call(t.Context(), "extractUsageOnComplete", proQuery, map[string]any{"status": "SUCCESS"}, payload)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"images_up_to_1_5k": float64(1), "images_above_1_5k": float64(1)}, alibabaObject(t, value))
|
||||
})
|
||||
|
||||
t.Run("failed and empty results do not complete a task", func(t *testing.T) {
|
||||
for _, payload := range []map[string]any{
|
||||
{"error": map[string]any{"code": "InvalidParameter", "message": "The parameter `seed` specified in the request is not valid. Request id: 0217", "param": "", "type": ""}},
|
||||
{"data": []any{map[string]any{"error": map[string]any{"code": "OutputImageSensitiveContentDetected", "message": "blocked"}}}, "usage": map[string]any{"generated_images": 0}},
|
||||
{"data": []any{}},
|
||||
} {
|
||||
_, err := plugin.Engine.Call(t.Context(), "parseSubmitResponse", map[string]any{"upstreamModel": lite, "publicTaskId": "task_public", "requestBody": map[string]any{"model": lite, "prompt": "a cat"}}, map[string]any{"statusCode": 200, "body": payload})
|
||||
require.Error(t, err)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("artifacts and Responses output use gateway URLs", func(t *testing.T) {
|
||||
value, err := plugin.Engine.Call(t.Context(), "listArtifacts", map[string]any{"taskId": "task_public", "status": "SUCCESS", "action": "text_to_image", "data": groupBody})
|
||||
require.NoError(t, err)
|
||||
encoded, err := common.Marshal(value)
|
||||
require.NoError(t, err)
|
||||
var artifacts []map[string]any
|
||||
require.NoError(t, common.Unmarshal(encoded, &artifacts))
|
||||
require.Len(t, artifacts, 2)
|
||||
assert.Equal(t, "image-1", artifacts[0]["key"])
|
||||
assert.Equal(t, "image", artifacts[0]["type"])
|
||||
assert.Equal(t, "image-2", artifacts[1]["key"])
|
||||
|
||||
value, err = plugin.Engine.Call(t.Context(), "buildContentRequest", map[string]any{"artifactKey": "image-2", "action": "text_to_image", "data": groupBody, "clientRequest": map[string]any{"method": "GET"}})
|
||||
require.NoError(t, err)
|
||||
content := alibabaObject(t, value)
|
||||
assert.Equal(t, second, content["url"])
|
||||
assert.Equal(t, true, content["credentialless"])
|
||||
|
||||
gateway := map[string]any{
|
||||
"model": lite,
|
||||
"artifacts": map[string]any{
|
||||
"image-1": map[string]any{"key": "image-1", "type": "image", "url": "https://gateway.example/v1/tasks/task_public/artifacts/image-1/content?access=a"},
|
||||
"image-2": map[string]any{"key": "image-2", "type": "image", "url": "https://gateway.example/v1/tasks/task_public/artifacts/image-2/content?access=b"},
|
||||
},
|
||||
}
|
||||
task := map[string]any{"task_id": "task_public", "status": "SUCCESS", "progress": "100%", "created_at": 10, "updated_at": 20, "data": groupBody}
|
||||
value, err = plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_responses", "renderFinal"}, gateway, task)
|
||||
require.NoError(t, err)
|
||||
machine := relay.NewPluginResponsesMachine("task_public", lite, 10, relay.DefaultPluginProtocolLimits())
|
||||
response, err := machine.FinalResponse(value, "SUCCESS")
|
||||
require.NoError(t, err)
|
||||
text := response["output"].([]any)[0].(map[string]any)["content"].([]any)[0].(map[string]any)["text"].(string)
|
||||
assert.Contains(t, text, "artifacts/image-1/content")
|
||||
assert.Contains(t, text, "artifacts/image-2/content")
|
||||
assert.NotContains(t, text, "ark-content.example")
|
||||
|
||||
_, err = plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_responses", "renderFinal"}, map[string]any{"model": lite}, task)
|
||||
require.ErrorContains(t, err, "image artifact is unavailable")
|
||||
})
|
||||
}
|
||||
|
||||
func TestDoubaoImageResponsesDecode(t *testing.T) {
|
||||
_, plugin := newDoubaoPlugin(t)
|
||||
const model = "doubao-seedream-5-0-lite-260128"
|
||||
decode := func(t *testing.T, body map[string]any) (map[string]any, error) {
|
||||
t.Helper()
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_responses", "decodeRequest"}, map[string]any{
|
||||
"model": body["model"], "stream": false, "body": map[string]any{"kind": "json", "value": body},
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return alibabaObject(t, value), nil
|
||||
}
|
||||
|
||||
resolved, err := decode(t, map[string]any{
|
||||
"model": model,
|
||||
"input": []any{map[string]any{"role": "user", "content": []any{
|
||||
map[string]any{"type": "input_text", "text": "a poster"},
|
||||
map[string]any{"type": "input_image", "image_url": "https://cdn.example/ref.png"},
|
||||
}}},
|
||||
"size": "2K",
|
||||
"sequential_image_generation": "auto",
|
||||
"sequential_image_generation_options": map[string]any{"max_images": 3},
|
||||
"watermark": false,
|
||||
// Host-owned background execution flag: must never become Ark's transparency option.
|
||||
"background": true,
|
||||
// Only Ark's web_search tool is forwarded, stripped to its type; function tools are dropped.
|
||||
"tools": []any{
|
||||
map[string]any{"type": "web_search", "search_context_size": "low"},
|
||||
map[string]any{"type": "function", "name": "lookup", "parameters": map[string]any{}},
|
||||
},
|
||||
"metadata": map[string]any{"ignored": true},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, model, resolved["model"])
|
||||
assert.Equal(t, "image_to_image", resolved["action"])
|
||||
request := resolved["requestBody"].(map[string]any)
|
||||
assert.Equal(t, map[string]any{
|
||||
"model": model, "prompt": "a poster", "image": []any{"https://cdn.example/ref.png"},
|
||||
"size": "2K", "sequential_image_generation": "auto", "sequential_image_generation_options": map[string]any{"max_images": float64(3)}, "watermark": false,
|
||||
"tools": []any{map[string]any{"type": "web_search"}},
|
||||
}, request)
|
||||
body, facts, url := submitDoubaoImage(t, plugin, "image_to_image", request)
|
||||
assert.Equal(t, doubaoBaseURL+"/api/v3/images/generations", url)
|
||||
assert.Equal(t, []any{"https://cdn.example/ref.png"}, body["image"])
|
||||
assert.NotContains(t, body, "background")
|
||||
assert.Equal(t, map[string]any{"images_up_to_1_5k": float64(0), "images_above_1_5k": float64(3), "input_images": float64(1), "layer_decomposition": false}, facts)
|
||||
|
||||
t.Run("function-only tools and background never reach a model without tool support", func(t *testing.T) {
|
||||
resolved, err := decode(t, map[string]any{"model": "doubao-seedream-4-0-250828", "input": "a cat", "background": true, "tools": []any{map[string]any{"type": "function", "name": "lookup"}}})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"model": "doubao-seedream-4-0-250828", "prompt": "a cat"}, resolved["requestBody"])
|
||||
})
|
||||
|
||||
_, err = decode(t, map[string]any{"model": model, "input": "a cat", "response_format": "b64_json"})
|
||||
require.ErrorContains(t, err, "response_format must be url")
|
||||
}
|
||||
|
||||
func decodeDoubaoOpenAIImage(t *testing.T, plugin *jsplugin.LoadedPlugin, operation, model string, body map[string]any) (map[string]any, error) {
|
||||
t.Helper()
|
||||
path := "/v1/images/generations"
|
||||
if operation == "edit" {
|
||||
path = "/v1/images/edits"
|
||||
}
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_image", "decodeRequest"}, map[string]any{
|
||||
"protocol": "openai_image", "operation": operation, "model": model, "path": path, "method": http.MethodPost, "body": body,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return alibabaObject(t, value), nil
|
||||
}
|
||||
|
||||
// OpenAI Images requests reach Ark with the same body, action and usage facts
|
||||
// as the equivalent native request. Host-owned and OpenAI-only fields are not
|
||||
// forwarded, and the facts never gain an image_count.
|
||||
func TestDoubaoOpenAIImageProtocol(t *testing.T) {
|
||||
registry, plugin := newDoubaoPlugin(t)
|
||||
const pro, lite, v40 = "doubao-seedream-5-0-pro-260628", "doubao-seedream-5-0-lite-260128", "doubao-seedream-4-0-250828"
|
||||
reference := "https://cdn.example/reference.png"
|
||||
facts := func(lower, higher, inputs float64, layered bool) map[string]any {
|
||||
return map[string]any{"images_up_to_1_5k": lower, "images_above_1_5k": higher, "input_images": inputs, "layer_decomposition": layered}
|
||||
}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
model string
|
||||
operation string
|
||||
native map[string]any // the equivalent native route body
|
||||
extra map[string]any // OpenAI fields accepted but not forwarded
|
||||
wantFacts map[string]any
|
||||
}{
|
||||
{"text to image", v40, "generate",
|
||||
map[string]any{"prompt": "a cat", "size": "2K", "seed": 42, "watermark": false},
|
||||
map[string]any{"n": 1, "response_format": "b64_json", "quality": "high", "user": "u1"}, facts(0, 1, 0, false)},
|
||||
{"1K reserves the lower tier", pro, "generate",
|
||||
map[string]any{"prompt": "a cat", "size": "1K", "output_format": "png"},
|
||||
map[string]any{"response_format": "url", "style": "vivid", "stream": false}, facts(1, 0, 0, false)},
|
||||
{"group generation with a reference image", lite, "edit",
|
||||
map[string]any{"prompt": "a brand kit", "image": reference, "size": "2K", "sequential_image_generation": "auto", "sequential_image_generation_options": map[string]any{"max_images": 4}, "tools": []any{map[string]any{"type": "web_search"}}},
|
||||
map[string]any{"n": nil}, facts(0, 4, 1, false)},
|
||||
{"layer decomposition", pro, "edit",
|
||||
map[string]any{"image": []any{reference}, "layer_decomposition": true, "size": "auto"},
|
||||
nil, facts(0, 17, 1, true)},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
nativeBody := map[string]any{"model": tc.model}
|
||||
maps.Copy(nativeBody, tc.native)
|
||||
native, err := decodeDoubaoImage(t, registry, plugin, nativeBody)
|
||||
require.NoError(t, err)
|
||||
openaiBody := map[string]any{"model": tc.model}
|
||||
maps.Copy(openaiBody, tc.native)
|
||||
maps.Copy(openaiBody, tc.extra)
|
||||
resolved, err := decodeDoubaoOpenAIImage(t, plugin, tc.operation, tc.model, map[string]any{"kind": "json", "value": openaiBody})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.model, resolved["model"])
|
||||
assert.Equal(t, native["action"], resolved["action"])
|
||||
assert.Equal(t, native["requestBody"], resolved["requestBody"])
|
||||
body, facts, url := submitDoubaoImage(t, plugin, resolved["action"].(string), resolved["requestBody"].(map[string]any))
|
||||
assert.Equal(t, doubaoBaseURL+"/api/v3/images/generations", url)
|
||||
for key := range tc.extra {
|
||||
assert.NotContains(t, body, key)
|
||||
}
|
||||
assert.Equal(t, tc.wantFacts, facts)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("multipart edits send uploads as Base64 data URLs", func(t *testing.T) {
|
||||
resolved, err := decodeDoubaoOpenAIImage(t, plugin, "edit", lite, map[string]any{"kind": "multipart",
|
||||
"fields": map[string][]string{
|
||||
"model": {lite}, "prompt": {"merge the two"}, "image": {reference}, "size": {"2K"}, "n": {"1"}, "response_format": {"b64_json"}, "watermark": {"false"}, "seed": {"7"},
|
||||
"sequential_image_generation": {"auto"}, "sequential_image_generation_options": {`{"max_images":2}`},
|
||||
},
|
||||
"files": []any{
|
||||
map[string]any{"ref": "request_file:image[]", "field": "image[]", "filename": "a.png", "mimeType": "image/png", "size": 10},
|
||||
map[string]any{"ref": "request_file:image[]#1", "field": "image[]", "filename": "b.jpg", "mimeType": "IMAGE/JPEG", "size": 10},
|
||||
},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "image_to_image", resolved["action"])
|
||||
images := []any{
|
||||
reference,
|
||||
map[string]any{"__fileRef": "request_file:image[]", "encoding": "dataUrl", "mimeType": "image/png", "maxBytes": float64(31457280)},
|
||||
map[string]any{"__fileRef": "request_file:image[]#1", "encoding": "dataUrl", "mimeType": "image/jpeg", "maxBytes": float64(31457280)},
|
||||
}
|
||||
assert.Equal(t, map[string]any{
|
||||
"model": lite, "prompt": "merge the two", "image": images, "size": "2K", "watermark": false, "seed": float64(7),
|
||||
"sequential_image_generation": "auto", "sequential_image_generation_options": map[string]any{"max_images": float64(2)},
|
||||
}, resolved["requestBody"])
|
||||
driver := map[string]any{"upstreamModel": lite, "model": lite, "action": "image_to_image", "requestBody": resolved["requestBody"], "baseUrl": doubaoBaseURL, "apiKey": "k", "usagePurpose": "facts"}
|
||||
value, err := plugin.Engine.Call(t.Context(), "buildSubmitRequest", driver)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, images, alibabaObject(t, value)["body"].(map[string]any)["image"], "placeholders reach the upstream body for the host to inline")
|
||||
value, err = plugin.Engine.Call(t.Context(), "extractUsage", driver)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, facts(0, 2, 3, false), alibabaObject(t, value))
|
||||
driver["usagePurpose"] = "billing_ratios"
|
||||
value, err = plugin.Engine.Call(t.Context(), "extractUsage", driver)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"image_count": float64(2)}, alibabaObject(t, value))
|
||||
})
|
||||
|
||||
t.Run("requests Ark cannot honor are rejected before billing", func(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
model string
|
||||
operation string
|
||||
body map[string]any
|
||||
wantErr string
|
||||
}{
|
||||
{"counts other than one", v40, "generate", map[string]any{"kind": "json", "value": map[string]any{"prompt": "a cat", "n": 2}}, "n must be 1"},
|
||||
{"multipart counts other than one", lite, "edit", map[string]any{"kind": "multipart", "fields": map[string][]string{"prompt": {"a cat"}, "image": {reference}, "n": {"4"}}}, "n must be 1"},
|
||||
{"streaming", v40, "generate", map[string]any{"kind": "json", "value": map[string]any{"prompt": "a cat", "stream": true}}, "stream is not supported"},
|
||||
{"unknown response formats", v40, "generate", map[string]any{"kind": "json", "value": map[string]any{"prompt": "a cat", "response_format": "base64"}}, "response_format must be url or b64_json"},
|
||||
{"edits without an image", v40, "edit", map[string]any{"kind": "json", "value": map[string]any{"prompt": "a cat"}}, "image is required"},
|
||||
{"masks", v40, "edit", map[string]any{"kind": "json", "value": map[string]any{"prompt": "a cat", "image": reference, "mask": reference}}, "mask is not supported"},
|
||||
{"mask uploads", v40, "edit", map[string]any{"kind": "multipart", "fields": map[string][]string{"prompt": {"a cat"}}, "files": []any{
|
||||
map[string]any{"ref": "request_file:image", "field": "image", "filename": "a.png", "mimeType": "image/png", "size": 10},
|
||||
map[string]any{"ref": "request_file:mask", "field": "mask", "filename": "m.png", "mimeType": "image/png", "size": 10},
|
||||
}}, "mask is not supported"},
|
||||
{"non-boolean multipart flags", v40, "edit", map[string]any{"kind": "multipart", "fields": map[string][]string{"prompt": {"a cat"}, "image": {reference}, "watermark": {"yes"}}}, "watermark must be true or false"},
|
||||
{"model capabilities", pro, "generate", map[string]any{"kind": "json", "value": map[string]any{"prompt": "a cat", "sequential_image_generation": "auto"}}, "sequential_image_generation is not supported"},
|
||||
} {
|
||||
_, err := decodeDoubaoOpenAIImage(t, plugin, tc.operation, tc.model, tc.body)
|
||||
require.ErrorContains(t, err, tc.wantErr, tc.name)
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("render returns the delivered image URLs", func(t *testing.T) {
|
||||
first, second := "https://ark-content.example/1.jpeg?sig=secret", "https://ark-content.example/2.jpeg?sig=secret"
|
||||
body := map[string]any{
|
||||
"model": lite, "created": 1789733151,
|
||||
"data": []any{
|
||||
map[string]any{"url": first, "size": "2848x1600"},
|
||||
map[string]any{"error": map[string]any{"code": "OutputImageSensitiveContentDetected", "message": "blocked"}},
|
||||
map[string]any{"url": second, "size": "2848x1600"},
|
||||
},
|
||||
"usage": map[string]any{"generated_images": 2},
|
||||
}
|
||||
value, err := plugin.Engine.CallPath(t.Context(), "protocols", []string{"openai_image", "render"}, map[string]any{"model": lite}, map[string]any{"task_id": "task_public", "status": "SUCCESS", "created_at": 1789733150, "data": body})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"created": float64(1789733150), "data": []any{map[string]any{"url": first}, map[string]any{"url": second}}}, alibabaObject(t, value))
|
||||
})
|
||||
}
|
||||
|
||||
// Channel model mapping may send a declared Seedream model to an Ark endpoint ID.
|
||||
// The declared model still drives capability checks and the image usage profile.
|
||||
func TestDoubaoImageEndpointMappingKeepsDeclaredModel(t *testing.T) {
|
||||
_, plugin := newDoubaoPlugin(t)
|
||||
const declared, endpoint = "doubao-seedream-5-0-pro-260628", "ep-20260918-seedream"
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
body map[string]any
|
||||
wantErr string
|
||||
}{
|
||||
{"declared capabilities still apply", map[string]any{"prompt": "a cat", "sequential_image_generation": "auto"}, "sequential_image_generation is not supported"},
|
||||
{"the endpoint receives the request", map[string]any{"prompt": "a cat", "size": "2K"}, ""},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
request := map[string]any{"model": declared}
|
||||
maps.Copy(request, tc.body)
|
||||
info := &relaycommon.RelayInfo{
|
||||
ChannelMeta: &relaycommon.ChannelMeta{ChannelBaseUrl: doubaoBaseURL, UpstreamModelName: endpoint},
|
||||
OriginModelName: declared,
|
||||
TaskRelayInfo: &relaycommon.TaskRelayInfo{PublicTaskID: "task_public", Action: "text_to_image"},
|
||||
}
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(info)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, doubaoImageRoute, nil)
|
||||
c.Set("task_request", request)
|
||||
taskErr := adaptor.ValidateRequestAndSetAction(c, info)
|
||||
if tc.wantErr != "" {
|
||||
require.NotNil(t, taskErr)
|
||||
assert.Equal(t, "plugin_request_invalid", taskErr.Code)
|
||||
assert.Contains(t, taskErr.Message, tc.wantErr)
|
||||
return
|
||||
}
|
||||
require.Nil(t, taskErr)
|
||||
reader, err := adaptor.BuildRequestBody(c, info)
|
||||
require.NoError(t, err)
|
||||
encoded, err := io.ReadAll(reader)
|
||||
require.NoError(t, err)
|
||||
var body map[string]any
|
||||
require.NoError(t, common.Unmarshal(encoded, &body))
|
||||
assert.Equal(t, endpoint, body["model"])
|
||||
facts, err := adaptor.ExtractUsageFactsValidated(c, info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"images_up_to_1_5k": float64(0), "images_above_1_5k": float64(1), "input_images": float64(0), "layer_decomposition": false}, alibabaObject(t, facts))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,20 @@
|
||||
package plugins_test
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/pkg/jsplugin"
|
||||
builtinplugins "github.com/QuantumNous/new-api/plugins"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
taskplugin "github.com/QuantumNous/new-api/relay/channel/task/jsplugin"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGoogleResponsesProtocol(t *testing.T) {
|
||||
testVideoResponsesProtocol(t, videoResponsesTestCase{
|
||||
@@ -28,3 +42,127 @@ func TestGoogleResponsesProtocol(t *testing.T) {
|
||||
wantVendorName: "gemini",
|
||||
})
|
||||
}
|
||||
|
||||
// A New API channel reaches Veo through another gateway's /google routes. The
|
||||
// gateway rebuilds the request this driver sent, names the operation after its
|
||||
// own task so it is polled there, and serves the video file only its Gemini
|
||||
// key can read.
|
||||
func TestGoogleThroughNewAPIGateway(t *testing.T) {
|
||||
source, err := builtinplugins.Source("google")
|
||||
require.NoError(t, err)
|
||||
plugin, err := jsplugin.NewRegistry().RegisterFactory(source, jsplugin.Options{Key: "google"})
|
||||
require.NoError(t, err)
|
||||
call := func(export string, members []string, args ...any) map[string]any {
|
||||
t.Helper()
|
||||
value, err := plugin.Engine.CallPath(t.Context(), export, members, args...)
|
||||
require.NoError(t, err)
|
||||
return decodePluginValue(t, value)
|
||||
}
|
||||
const veo = "veo-3.1-fast-generate-preview"
|
||||
gateway := map[string]any{"kind": "new_api"}
|
||||
submitParams := map[string]any{"model": veo + ":predictLongRunning"}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
requestBody map[string]any
|
||||
}{
|
||||
{"text with parameters", map[string]any{"model": veo, "prompt": "p", "duration": 6, "metadata": map[string]any{"resolution": "1080p", "aspectRatio": "9:16", "negativePrompt": "rain"}}},
|
||||
{"first frame", map[string]any{"model": veo, "prompt": "p", "images": []any{"data:image/png;base64,aGVsbG8="}, "size": "1280x720"}},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
downstream := map[string]any{"requestBody": tc.requestBody, "model": veo, "upstreamModel": veo,
|
||||
"baseUrl": "https://gateway.example", "apiKey": "sk-gateway", "upstream": gateway}
|
||||
sent := call("buildSubmitRequest", nil, downstream)
|
||||
assert.Equal(t, "https://gateway.example/google/v1beta/models/"+veo+":predictLongRunning", sent["url"])
|
||||
assert.Equal(t, map[string]any{"Accept": "application/json", "Content-Type": "application/json", "Authorization": "Bearer sk-gateway"}, sent["headers"])
|
||||
|
||||
intent := call("native", []string{"predictLongRunning"}, map[string]any{"params": submitParams, "body": map[string]any{"kind": "json", "value": sent["body"]}})
|
||||
assert.Equal(t, veo, intent["model"])
|
||||
assert.Equal(t, sent["action"], intent["action"])
|
||||
upstream := map[string]any{"requestBody": intent["requestBody"], "model": veo, "upstreamModel": veo,
|
||||
"baseUrl": "https://generativelanguage.example", "apiKey": "vendor-key", "upstream": map[string]any{"kind": "vendor"}}
|
||||
forwarded := call("buildSubmitRequest", nil, upstream)
|
||||
assert.Equal(t, "https://generativelanguage.example/v1beta/models/"+veo+":predictLongRunning", forwarded["url"])
|
||||
assert.Equal(t, "vendor-key", forwarded["headers"].(map[string]any)["x-goog-api-key"])
|
||||
assert.Equal(t, sent["body"], forwarded["body"])
|
||||
assert.Equal(t, call("extractUsage", nil, downstream), call("extractUsage", nil, upstream))
|
||||
})
|
||||
}
|
||||
|
||||
created := call("native", []string{"operation"}, map[string]any{"params": submitParams},
|
||||
map[string]any{"task_id": "task_pub", "status": "SUBMITTED", "data": map[string]any{"name": "models/" + veo + "/operations/op-private"}})
|
||||
assert.Equal(t, map[string]any{"name": "models/" + veo + "/operations/task_pub"}, created)
|
||||
upstreamTaskID := call("parseSubmitResponse", nil, map[string]any{}, map[string]any{"body": created})["taskId"]
|
||||
query := call("buildQueryRequest", nil, map[string]any{"taskId": upstreamTaskID, "baseUrl": "https://gateway.example", "apiKey": "sk-gateway", "upstream": gateway})
|
||||
assert.Equal(t, "https://gateway.example/google/v1beta/models/"+veo+"/operations/task_pub", query["url"])
|
||||
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(&relaycommon.RelayInfo{ChannelMeta: &relaycommon.ChannelMeta{
|
||||
ChannelType: constant.ChannelTypeNewAPI, ApiKey: "sk-gateway", ChannelBaseUrl: "https://gateway.example",
|
||||
}})
|
||||
queryParams := map[string]any{"model": veo, "task_id": "task_pub"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
task map[string]any
|
||||
wantStatus string
|
||||
wantReason string
|
||||
}{
|
||||
{
|
||||
name: "running", wantStatus: "IN_PROGRESS",
|
||||
task: map[string]any{"status": "IN_PROGRESS", "data": map[string]any{"name": "models/" + veo + "/operations/op-private"}},
|
||||
},
|
||||
{
|
||||
name: "succeeded", wantStatus: "SUCCESS",
|
||||
task: map[string]any{"status": "SUCCESS", "data": map[string]any{"name": "models/" + veo + "/operations/op-private", "done": true,
|
||||
"response": map[string]any{"generateVideoResponse": map[string]any{"generatedVideos": []any{
|
||||
map[string]any{"video": map[string]any{"uri": "https://generativelanguage.example/v1beta/files/f1:download?alt=media"}},
|
||||
}}}}},
|
||||
},
|
||||
{
|
||||
name: "failed by the gateway poller", wantStatus: "FAILURE", wantReason: "task timed out",
|
||||
task: map[string]any{"status": "FAILURE", "fail_reason": "task timed out", "data": map[string]any{"name": "models/" + veo + "/operations/op-private"}},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.task["task_id"] = "task_pub"
|
||||
rendered := call("native", []string{"operation"}, map[string]any{"params": queryParams}, tc.task)
|
||||
assert.Equal(t, "models/"+veo+"/operations/task_pub", rendered["name"])
|
||||
result := call("parseTaskResult", nil, map[string]any{}, rendered)
|
||||
assert.Equal(t, tc.wantStatus, result["status"])
|
||||
assert.Equal(t, tc.wantReason, common.Interface2String(result["reason"]))
|
||||
if tc.wantStatus != "SUCCESS" {
|
||||
return
|
||||
}
|
||||
data, err := common.Marshal(rendered)
|
||||
require.NoError(t, err)
|
||||
task := &model.Task{TaskID: "task_down", Status: model.TaskStatusSuccess, Data: data, PrivateData: model.TaskPrivateData{UpstreamTaskID: upstreamTaskID.(string)}}
|
||||
artifacts, err := adaptor.ListArtifacts(task)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []channel.TaskArtifact{{Key: "video", Type: "video"}}, artifacts)
|
||||
descriptor, err := adaptor.BuildContentRequest(t.Context(), task, "video", channel.TaskArtifactClientRequest{Method: http.MethodGet})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, &channel.TaskContentRequest{
|
||||
URL: "https://gateway.example/v1/tasks/task_pub/artifacts/video/content",
|
||||
Method: http.MethodGet,
|
||||
Headers: map[string]string{"Authorization": "Bearer sk-gateway"},
|
||||
}, descriptor)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("native bodies are checked before billing", func(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
model string
|
||||
body map[string]any
|
||||
wantErr string
|
||||
}{
|
||||
{veo + ":generateVideos", map[string]any{"instances": []any{map[string]any{"prompt": "p"}}}, "models/{model}:predictLongRunning"},
|
||||
{veo + ":predictLongRunning", map[string]any{"instances": []any{map[string]any{"prompt": "p", "lastFrame": map[string]any{}}}}, "instances[0].lastFrame is not supported"},
|
||||
{veo + ":predictLongRunning", map[string]any{"instances": []any{map[string]any{"prompt": "p"}}, "parameters": map[string]any{"durationSeconds": 5}}, "durationSeconds must be one of 4, 6, or 8"},
|
||||
} {
|
||||
_, err := plugin.Engine.CallPath(t.Context(), "native", []string{"predictLongRunning"}, map[string]any{
|
||||
"params": map[string]any{"model": tc.model}, "body": map[string]any{"kind": "json", "value": tc.body},
|
||||
})
|
||||
require.ErrorContains(t, err, tc.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -8,12 +8,14 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/pkg/jsplugin"
|
||||
builtinplugins "github.com/QuantumNous/new-api/plugins"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
taskplugin "github.com/QuantumNous/new-api/relay/channel/task/jsplugin"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
@@ -111,33 +113,55 @@ func TestHailuoUsageProfilesPerModel(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
// MiniMax serves v1 videos only through the signed download_url of
|
||||
// /v1/files/retrieve; that link is signed for GET, so HEAD clients get GET too.
|
||||
func TestHailuoArtifactContentProxy(t *testing.T) {
|
||||
source, err := builtinplugins.Source("hailuo")
|
||||
require.NoError(t, err)
|
||||
plugin, err := jsplugin.NewRegistry().RegisterFactory(source, jsplugin.Options{Key: "hailuo"})
|
||||
require.NoError(t, err)
|
||||
adaptor := taskplugin.New(plugin)
|
||||
service.InitHttpClient()
|
||||
const signedURL = "https://bucket.oss-cn-wulanchabu.aliyuncs.com/video.mp4?Expires=1&OSSAccessKeyId=key&Signature=sig"
|
||||
retrieveBodies := map[string]string{
|
||||
"file/with space": `{"file":{"file_id":1,"download_url":"` + signedURL + `"},"base_resp":{"status_code":0,"status_msg":"success"}}`,
|
||||
"login": `{"base_resp":{"status_code":1004,"status_msg":"login fail"}}`,
|
||||
"missing": `{"file":{"file_id":3},"base_resp":{"status_code":0,"status_msg":"success"}}`,
|
||||
"html": `<html>gateway error</html>`,
|
||||
}
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, http.MethodGet, r.Method)
|
||||
assert.Equal(t, "/v1/files/retrieve", r.URL.Path)
|
||||
assert.Equal(t, "Bearer test-ak", r.Header.Get("Authorization"))
|
||||
_, _ = w.Write([]byte(retrieveBodies[r.URL.Query().Get("file_id")]))
|
||||
}))
|
||||
defer upstream.Close()
|
||||
adaptor := taskplugin.New(loadHailuoPlugin(t))
|
||||
adaptor.Init(&relaycommon.RelayInfo{
|
||||
ChannelMeta: &relaycommon.ChannelMeta{
|
||||
ApiKey: "test-ak",
|
||||
ChannelBaseUrl: "https://api.minimax.example",
|
||||
},
|
||||
ChannelMeta: &relaycommon.ChannelMeta{ApiKey: "test-ak", ChannelBaseUrl: upstream.URL},
|
||||
})
|
||||
data, err := common.Marshal(map[string]any{"file_id": "file/with space"})
|
||||
require.NoError(t, err)
|
||||
task := &model.Task{TaskID: "task-public", Status: model.TaskStatusSuccess, Data: data}
|
||||
|
||||
artifacts, err := adaptor.ListArtifacts(task)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []channel.TaskArtifact{{Key: "video", Type: "video", MimeType: "video/mp4"}}, artifacts)
|
||||
for _, tc := range []struct {
|
||||
fileID string
|
||||
wantErr string
|
||||
}{
|
||||
{fileID: "file/with space"},
|
||||
{fileID: "login", wantErr: "MiniMax file retrieve failed: login fail"},
|
||||
{fileID: "missing", wantErr: "artifact_not_found"},
|
||||
{fileID: "html", wantErr: "MiniMax file retrieve returned an unexpected response: HTTP 200"},
|
||||
} {
|
||||
t.Run(tc.fileID, func(t *testing.T) {
|
||||
data, err := common.Marshal(map[string]any{"file_id": tc.fileID})
|
||||
require.NoError(t, err)
|
||||
task := &model.Task{TaskID: "task-public", Status: model.TaskStatusSuccess, Data: data}
|
||||
artifacts, err := adaptor.ListArtifacts(task)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []channel.TaskArtifact{{Key: "video", Type: "video", MimeType: "video/mp4"}}, artifacts)
|
||||
|
||||
descriptor, err := adaptor.BuildContentRequest(task, "video", channel.TaskArtifactClientRequest{Method: http.MethodHead})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, descriptor)
|
||||
assert.Equal(t, "https://api.minimax.example/v1/files/download?file_id=file%2Fwith%20space", descriptor.URL)
|
||||
assert.Equal(t, http.MethodHead, descriptor.Method)
|
||||
assert.Equal(t, map[string]string{"Accept": "video/*", "Authorization": "Bearer test-ak"}, descriptor.Headers)
|
||||
assert.False(t, descriptor.Credentialless)
|
||||
descriptor, err := adaptor.BuildContentRequest(t.Context(), task, "video", channel.TaskArtifactClientRequest{Method: http.MethodHead})
|
||||
if tc.wantErr != "" {
|
||||
require.ErrorContains(t, err, tc.wantErr)
|
||||
return
|
||||
}
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, &channel.TaskContentRequest{URL: signedURL, Method: http.MethodGet, Credentialless: true}, descriptor)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func loadHailuoPlugin(t *testing.T) *jsplugin.LoadedPlugin {
|
||||
@@ -386,6 +410,11 @@ func TestHailuoQueryRequestEndpointByModel(t *testing.T) {
|
||||
ctx: map[string]any{"taskId": "t1", "model": "h3", "upstreamModel": "MiniMax-H3"},
|
||||
want: "https://api.minimax.example/v2/query/video_generation/t1",
|
||||
},
|
||||
{
|
||||
name: "a New API gateway serves H3 on the prefixed v2 route",
|
||||
ctx: map[string]any{"taskId": "t1", "upstreamModel": "MiniMax-H3", "upstream": map[string]any{"kind": "new_api"}},
|
||||
want: "https://api.minimax.example/hailuo/v2/query/video_generation/t1",
|
||||
},
|
||||
}
|
||||
for _, testCase := range testCases {
|
||||
t.Run(testCase.name, func(t *testing.T) {
|
||||
@@ -554,7 +583,7 @@ func TestHailuoH3ArtifactContentProxy(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []channel.TaskArtifact{{Key: "video", Type: "video", MimeType: "video/mp4"}}, artifacts)
|
||||
|
||||
descriptor, err := adaptor.BuildContentRequest(task, "video", channel.TaskArtifactClientRequest{Method: http.MethodGet})
|
||||
descriptor, err := adaptor.BuildContentRequest(t.Context(), task, "video", channel.TaskArtifactClientRequest{Method: http.MethodGet})
|
||||
require.NoError(t, err)
|
||||
require.NotNil(t, descriptor)
|
||||
assert.Equal(t, "https://cdn.example/h3.mp4", descriptor.URL)
|
||||
@@ -624,3 +653,176 @@ func TestHailuoH3PassesOpenAIVideoDecode(t *testing.T) {
|
||||
require.True(t, ok)
|
||||
assert.Equal(t, float64(12), requestBody["duration"])
|
||||
}
|
||||
|
||||
// A New API channel reaches this plugin on another gateway. The downstream
|
||||
// driver addresses the gateway's /hailuo routes, whose decoders give the
|
||||
// gateway's driver the same request and whose presenters give the downstream
|
||||
// driver the same task state, so MiniMax receives one body and both gateways
|
||||
// settle on the same facts.
|
||||
func TestHailuoThroughNewAPIGateway(t *testing.T) {
|
||||
plugin := loadHailuoPlugin(t)
|
||||
call := func(export string, members []string, args ...any) map[string]any {
|
||||
t.Helper()
|
||||
value, err := plugin.Engine.CallPath(t.Context(), export, members, args...)
|
||||
require.NoError(t, err)
|
||||
encoded, err := common.Marshal(value)
|
||||
require.NoError(t, err)
|
||||
var decoded map[string]any
|
||||
require.NoError(t, common.Unmarshal(encoded, &decoded))
|
||||
return decoded
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
model string
|
||||
requestBody map[string]any
|
||||
route string
|
||||
decoder string
|
||||
vendorPath string
|
||||
}{
|
||||
{
|
||||
name: "v1 image to video",
|
||||
model: "MiniMax-Hailuo-02",
|
||||
requestBody: map[string]any{"model": "MiniMax-Hailuo-02", "prompt": "p", "duration": 10, "size": "512P", "metadata": map[string]any{
|
||||
"first_frame_image": "https://cdn.example/first.png", "prompt_optimizer": false,
|
||||
}},
|
||||
route: "/hailuo/v1/video_generation",
|
||||
decoder: "createTask",
|
||||
vendorPath: "/v1/video_generation",
|
||||
},
|
||||
{
|
||||
name: "H3 reference video",
|
||||
model: "MiniMax-H3",
|
||||
requestBody: map[string]any{"model": "MiniMax-H3", "prompt": "p", "duration": 8, "size": "2K", "metadata": map[string]any{
|
||||
"reference_video": "https://cdn.example/ref.mp4", "ratio": "9:16", "aigc_watermark": true,
|
||||
}},
|
||||
route: "/hailuo/v2/video_generation",
|
||||
decoder: "createH3Task",
|
||||
vendorPath: "/v2/video_generation",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
downstream := map[string]any{"requestBody": tc.requestBody, "model": tc.model, "upstreamModel": tc.model,
|
||||
"baseUrl": "https://gateway.example", "apiKey": "sk-gateway", "upstream": map[string]any{"kind": "new_api"}}
|
||||
sent := call("buildSubmitRequest", nil, downstream)
|
||||
assert.Equal(t, "https://gateway.example"+tc.route, sent["url"])
|
||||
|
||||
intent := call("native", []string{tc.decoder}, map[string]any{"body": map[string]any{"kind": "json", "value": sent["body"]}})
|
||||
assert.Equal(t, tc.model, intent["model"])
|
||||
assert.Equal(t, sent["action"], intent["action"])
|
||||
upstream := map[string]any{"requestBody": intent["requestBody"], "model": tc.model, "upstreamModel": tc.model,
|
||||
"baseUrl": "https://api.minimax.example", "apiKey": "vendor-key", "upstream": map[string]any{"kind": "vendor"}}
|
||||
forwarded := call("buildSubmitRequest", nil, upstream)
|
||||
assert.Equal(t, "https://api.minimax.example"+tc.vendorPath, forwarded["url"])
|
||||
assert.Equal(t, sent["body"], forwarded["body"])
|
||||
assert.Equal(t, call("extractUsage", nil, downstream), call("extractUsage", nil, upstream))
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("create responses", func(t *testing.T) {
|
||||
for model, data := range map[string]map[string]any{
|
||||
"MiniMax-Hailuo-02": {"task_id": "task_pub", "base_resp": map[string]any{"status_code": 0, "status_msg": "success"}},
|
||||
"MiniMax-H3": {"task_id": "task_pub"},
|
||||
} {
|
||||
created := call("native", []string{"taskCreated"}, map[string]any{}, map[string]any{"task_id": "task_pub", "status": "SUBMITTED", "data": data})
|
||||
parsed := call("parseSubmitResponse", nil, map[string]any{"upstreamModel": model}, map[string]any{"body": created})
|
||||
assert.Equal(t, "task_pub", parsed["taskId"], model)
|
||||
}
|
||||
})
|
||||
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(&relaycommon.RelayInfo{ChannelMeta: &relaycommon.ChannelMeta{
|
||||
ChannelType: constant.ChannelTypeNewAPI, ApiKey: "sk-gateway", ChannelBaseUrl: "https://gateway.example",
|
||||
}})
|
||||
success := map[string]any{"status_code": 0, "status_msg": "success"}
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
presenter string
|
||||
task map[string]any
|
||||
wantStatus string
|
||||
wantReason string
|
||||
wantContent *channel.TaskContentRequest
|
||||
}{
|
||||
{
|
||||
name: "v1 before the first poll", presenter: "taskStatus", wantStatus: "IN_PROGRESS",
|
||||
task: map[string]any{"status": "SUBMITTED", "data": map[string]any{"task_id": "task_pub", "base_resp": success}},
|
||||
},
|
||||
{
|
||||
name: "v1 succeeded", presenter: "taskStatus", wantStatus: "SUCCESS",
|
||||
task: map[string]any{"status": "SUCCESS", "data": map[string]any{
|
||||
"task_id": "task_pub", "status": "Success", "file_id": "f1", "video_width": 1920, "video_height": 1080, "base_resp": success,
|
||||
}},
|
||||
wantContent: &channel.TaskContentRequest{
|
||||
URL: "https://gateway.example/v1/tasks/task_pub/artifacts/video/content",
|
||||
Method: http.MethodHead,
|
||||
Headers: map[string]string{"Authorization": "Bearer sk-gateway"},
|
||||
},
|
||||
},
|
||||
{
|
||||
name: "v1 failed by the gateway poller", presenter: "taskStatus", wantStatus: "FAILURE", wantReason: "task failed",
|
||||
task: map[string]any{"status": "FAILURE", "fail_reason": "task timed out", "data": map[string]any{"task_id": "task_pub", "status": "Processing", "base_resp": success}},
|
||||
},
|
||||
{
|
||||
name: "H3 before the first poll", presenter: "h3TaskStatus", wantStatus: "QUEUED",
|
||||
task: map[string]any{"status": "SUBMITTED", "data": map[string]any{"task_id": "task_pub"}},
|
||||
},
|
||||
{
|
||||
name: "H3 succeeded", presenter: "h3TaskStatus", wantStatus: "SUCCESS",
|
||||
task: map[string]any{"status": "SUCCESS", "data": map[string]any{"task": map[string]any{
|
||||
"id": "task_pub", "status": "succeeded", "resolution": "2K", "content": map[string]any{"url": "https://cdn.example/h3.mp4"},
|
||||
"usage": map[string]any{"output_seconds": 8, "input_seconds": 7.5, "input_image_count": 0},
|
||||
}}},
|
||||
wantContent: &channel.TaskContentRequest{URL: "https://cdn.example/h3.mp4", Method: http.MethodHead, Credentialless: true},
|
||||
},
|
||||
{
|
||||
name: "H3 failed by the gateway poller", presenter: "h3TaskStatus", wantStatus: "FAILURE", wantReason: "task timed out",
|
||||
task: map[string]any{"status": "FAILURE", "fail_reason": "task timed out", "data": map[string]any{"task": map[string]any{"id": "task_pub", "status": "running"}}},
|
||||
},
|
||||
{
|
||||
name: "H3 failed by MiniMax", presenter: "h3TaskStatus", wantStatus: "FAILURE", wantReason: "sensitive content",
|
||||
task: map[string]any{"status": "FAILURE", "fail_reason": "sensitive content", "data": map[string]any{"task": map[string]any{
|
||||
"id": "task_pub", "status": "failed", "error": map[string]any{"code": "1026", "message": "sensitive content"},
|
||||
}}},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.task["task_id"] = "task_pub"
|
||||
rendered := call("native", []string{tc.presenter}, map[string]any{}, tc.task)
|
||||
result := call("parseTaskResult", nil, map[string]any{}, rendered, map[string]any{"status": http.StatusOK})
|
||||
assert.Equal(t, tc.wantStatus, result["status"])
|
||||
assert.Equal(t, tc.wantReason, common.Interface2String(result["reason"]))
|
||||
if tc.wantContent == nil {
|
||||
return
|
||||
}
|
||||
facts := call("extractUsageOnComplete", nil, nil, nil, rendered)
|
||||
assert.NotEmpty(t, facts)
|
||||
assert.Equal(t, call("extractUsageOnComplete", nil, nil, nil, tc.task["data"]), facts)
|
||||
|
||||
data, err := common.Marshal(rendered)
|
||||
require.NoError(t, err)
|
||||
task := &model.Task{TaskID: "task_down", Status: model.TaskStatusSuccess, Data: data, PrivateData: model.TaskPrivateData{UpstreamTaskID: "task_pub"}}
|
||||
artifacts, err := adaptor.ListArtifacts(task)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, []channel.TaskArtifact{{Key: "video", Type: "video", MimeType: "video/mp4"}}, artifacts)
|
||||
descriptor, err := adaptor.BuildContentRequest(t.Context(), task, "video", channel.TaskArtifactClientRequest{Method: http.MethodHead})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantContent, descriptor)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("native bodies are checked before billing", func(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
decoder string
|
||||
body map[string]any
|
||||
wantErr string
|
||||
}{
|
||||
{"createTask", map[string]any{"model": "MiniMax-Hailuo-2.3", "prompt": "p", "duration": 10, "resolution": "1080P"}, "duration 10 only allows resolution 768P"},
|
||||
{"createTask", map[string]any{"model": "MiniMax-Hailuo-2.3-Fast", "prompt": "p"}, "supports image-to-video only"},
|
||||
{"createH3Task", map[string]any{"model": "MiniMax-H3", "prompt": "p"}, "content must be an array"},
|
||||
{"createH3Task", map[string]any{"model": "MiniMax-H3", "content": []any{map[string]any{"type": "image_url", "image_url": map[string]any{"url": "u"}}}}, "requires a non-empty text item"},
|
||||
} {
|
||||
_, err := plugin.Engine.CallPath(t.Context(), "native", []string{tc.decoder}, map[string]any{"body": map[string]any{"kind": "json", "value": tc.body}})
|
||||
require.ErrorContains(t, err, tc.wantErr)
|
||||
}
|
||||
})
|
||||
}
|
||||
+653
-104
File diff suppressed because it is too large.
Load diff
+780
-71
File diff suppressed because it is too large.
Load diff
@@ -7,11 +7,12 @@ export const meta = {
|
||||
en: "Google Veo video generation on the Gemini API (text-to-video and image-to-video)",
|
||||
zh: "Google Veo 视频生成（文生视频、图生视频），Gemini API 版本",
|
||||
},
|
||||
version: "1.0.2",
|
||||
version: "1.1.0",
|
||||
author: { name: "QuantumNous" },
|
||||
channelTypes: [24],
|
||||
models: ["veo-3.0-generate-001", "veo-3.0-fast-generate-001", "veo-3.1-generate-preview", "veo-3.1-fast-generate-preview"],
|
||||
fetchMode: "per_task",
|
||||
upstreams: ["vendor", "new_api"],
|
||||
usageSchema: {
|
||||
// Requested video duration in seconds. Allowed values: 4, 6, 8.
|
||||
seconds: {
|
||||
@@ -32,6 +33,14 @@ export const meta = {
|
||||
{ label: "8s × 4k", facts: { seconds: 8, resolution: "4k" } },
|
||||
{ label: "4s × 720p", facts: { seconds: 4, resolution: "720p" } },
|
||||
],
|
||||
// Gemini API request and response bodies, for clients and for a downstream
|
||||
// New API gateway. The model segment carries the method,
|
||||
// models/{model}:predictLongRunning, and the operation it returns is named
|
||||
// after the gateway task.
|
||||
routes: [
|
||||
{ method: "POST", path: "/google/v1beta/models/:model", type: "submit", decode: "predictLongRunning", render: "operation" },
|
||||
{ method: "GET", path: "/google/v1beta/models/:model/operations/:task_id", type: "query", render: "operation" },
|
||||
],
|
||||
protocols: [{ name: "openai_responses", supports: ["stream", "sync", "background"] }, "openai_video"],
|
||||
};
|
||||
|
||||
@@ -39,6 +48,25 @@ function trimmed(value) {
|
||||
return String(value || "").trim();
|
||||
}
|
||||
|
||||
function plainObject(value) {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
||||
}
|
||||
|
||||
// On a New API channel the same plugin answers on another gateway, which
|
||||
// serves the Gemini API wire format on the prefixed native routes and takes
|
||||
// its own token as a Bearer header.
|
||||
function viaGateway(ctx) {
|
||||
return Boolean(ctx.upstream && ctx.upstream.kind === "new_api");
|
||||
}
|
||||
|
||||
function apiRoot(ctx) {
|
||||
return ctx.baseUrl + (viaGateway(ctx) ? "/google" : "");
|
||||
}
|
||||
|
||||
function authHeaders(ctx) {
|
||||
return viaGateway(ctx) ? { Authorization: "Bearer " + ctx.apiKey } : { "x-goog-api-key": ctx.apiKey };
|
||||
}
|
||||
|
||||
function responsesInput(req) {
|
||||
const texts = [],
|
||||
images = [];
|
||||
@@ -160,12 +188,64 @@ function version(ctx) {
|
||||
return settings.geminiVersion || "v1beta";
|
||||
}
|
||||
|
||||
export const native = {
|
||||
predictLongRunning: function (ctx) {
|
||||
const segment = String((ctx.params || {}).model || "");
|
||||
const separator = segment.lastIndexOf(":");
|
||||
if (separator <= 0 || segment.slice(separator + 1) !== "predictLongRunning") throw new Error("the path must name models/{model}:predictLongRunning");
|
||||
const model = segment.slice(0, separator);
|
||||
if (!ctx.body || ctx.body.kind !== "json") throw new Error("JSON body required");
|
||||
const body = plainObject(ctx.body.value);
|
||||
if (!body) throw new Error("request body must be an object");
|
||||
const instances = Array.isArray(body.instances) ? body.instances : [];
|
||||
const instance = plainObject(instances[0]);
|
||||
if (instances.length !== 1 || !instance) throw new Error("instances must hold exactly one instance");
|
||||
// buildSubmitRequest sends only the prompt and the first frame.
|
||||
for (const key of Object.keys(instance)) {
|
||||
if (key !== "prompt" && key !== "image") throw new Error("instances[0]." + key + " is not supported");
|
||||
}
|
||||
const metadata = Object.assign({}, plainObject(body.parameters) || {});
|
||||
const requestBody = { model: model, prompt: instance.prompt, metadata: metadata };
|
||||
if (metadata.durationSeconds !== undefined) {
|
||||
requestBody.duration = metadata.durationSeconds;
|
||||
delete metadata.durationSeconds;
|
||||
if ([4, 6, 8].indexOf(Number(requestBody.duration)) < 0) throw new Error("durationSeconds must be one of 4, 6, or 8");
|
||||
}
|
||||
if (metadata.resolution !== undefined && ["720p", "1080p", "4k"].indexOf(String(metadata.resolution).toLowerCase()) < 0)
|
||||
throw new Error("resolution must be one of 720p, 1080p, or 4k");
|
||||
if (instance.image !== undefined) {
|
||||
const inline = plainObject(plainObject(instance.image) && instance.image.inlineData);
|
||||
if (!inline || !trimmed(inline.data)) throw new Error("instances[0].image must carry inlineData");
|
||||
requestBody.images = ["data:" + (trimmed(inline.mimeType) || "application/octet-stream") + ";base64," + inline.data];
|
||||
}
|
||||
return { kind: "submit", model: model, action: requestBody.images ? "image_to_video" : "text_to_video", requestBody: requestBody };
|
||||
},
|
||||
// The operation is named after the gateway task, so the caller polls it
|
||||
// here. The poller can fail a task whose operation was still running; that
|
||||
// operation is rendered done with the failure.
|
||||
operation: function (ctx, task) {
|
||||
const data = plainObject(task.data) || {};
|
||||
const named = /^models\/([^/]+)\/operations\//.exec(String(data.name || ""));
|
||||
const model = named ? named[1] : String((ctx.params || {}).model || "").split(":")[0];
|
||||
const output = Object.assign({}, data, { name: "models/" + model + "/operations/" + task.task_id });
|
||||
if (task.status === "FAILURE" && !(plainObject(data.error) && data.error.message)) {
|
||||
output.done = true;
|
||||
output.error = { message: task.fail_reason || "task failed" };
|
||||
delete output.response;
|
||||
}
|
||||
return output;
|
||||
},
|
||||
error: function (ctx, error) {
|
||||
return { error: { code: error.httpStatus, message: error.message } };
|
||||
},
|
||||
};
|
||||
|
||||
export function buildSubmitRequest(ctx) {
|
||||
const result = converted(ctx);
|
||||
return {
|
||||
url: ctx.baseUrl + "/" + version(ctx) + "/models/" + ctx.upstreamModel + ":predictLongRunning",
|
||||
url: apiRoot(ctx) + "/" + version(ctx) + "/models/" + ctx.upstreamModel + ":predictLongRunning",
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", "x-goog-api-key": ctx.apiKey },
|
||||
headers: Object.assign({ "Content-Type": "application/json", Accept: "application/json" }, authHeaders(ctx)),
|
||||
body: result.body,
|
||||
action: result.action,
|
||||
};
|
||||
@@ -198,9 +278,9 @@ export function extractUsageOnComplete() {
|
||||
|
||||
export function buildQueryRequest(ctx) {
|
||||
return {
|
||||
url: ctx.baseUrl + "/v1beta/" + utils.base64URLDecode(ctx.taskId),
|
||||
url: apiRoot(ctx) + "/v1beta/" + utils.base64URLDecode(ctx.taskId),
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json", "x-goog-api-key": ctx.apiKey },
|
||||
headers: Object.assign({ Accept: "application/json" }, authHeaders(ctx)),
|
||||
};
|
||||
}
|
||||
|
||||
@@ -235,6 +315,16 @@ export function buildContentRequest(ctx) {
|
||||
if (ctx.artifactKey !== "video") throw new Error("artifact_not_found");
|
||||
const url = artifactVideoURL(ctx);
|
||||
if (!url) throw new Error("artifact_not_found");
|
||||
// The file needs the Gemini key, which only the gateway holds; it serves the
|
||||
// video as the artifact of its task, the last segment of the operation name.
|
||||
if (viaGateway(ctx)) {
|
||||
const taskID = utils.base64URLDecode(ctx.upstreamTaskId).split("/").pop();
|
||||
return {
|
||||
url: ctx.baseUrl + "/v1/tasks/" + encodeURIComponent(taskID) + "/artifacts/video/content",
|
||||
method: ctx.clientRequest.method,
|
||||
headers: { Authorization: "Bearer " + ctx.apiKey },
|
||||
};
|
||||
}
|
||||
return { url: url, method: ctx.clientRequest.method, headers: { "x-goog-api-key": ctx.apiKey } };
|
||||
}
|
||||
|
||||
|
||||
+178
-45
@@ -53,6 +53,33 @@ const HAILUO_01_USAGE_SCHEMA = {
|
||||
seconds: VIDEO_SECONDS_FIELD,
|
||||
};
|
||||
|
||||
// MiniMax serves MiniMax-H3 on its /v2 video API and every other model on /v1.
|
||||
const H3_MODEL = "MiniMax-H3";
|
||||
const V1_MODELS = [
|
||||
"MiniMax-Hailuo-2.3",
|
||||
"MiniMax-Hailuo-2.3-Fast",
|
||||
"MiniMax-Hailuo-02",
|
||||
"T2V-01-Director",
|
||||
"T2V-01",
|
||||
"I2V-01-Director",
|
||||
"I2V-01-live",
|
||||
"I2V-01",
|
||||
"S2V-01",
|
||||
];
|
||||
|
||||
// Vendor fields the OpenAI-style requests carry in metadata; the MiniMax body
|
||||
// has them at the top level.
|
||||
const V1_METADATA_FIELDS = [
|
||||
"prompt_optimizer",
|
||||
"fast_pretreatment",
|
||||
"callback_url",
|
||||
"aigc_watermark",
|
||||
"first_frame_image",
|
||||
"last_frame_image",
|
||||
"subject_reference",
|
||||
];
|
||||
const H3_METADATA_FIELDS = ["callback_url", "aigc_watermark"];
|
||||
|
||||
export const meta = {
|
||||
apiVersion: 1,
|
||||
key: "hailuo",
|
||||
@@ -62,22 +89,12 @@ export const meta = {
|
||||
en: "MiniMax Hailuo video generation (text-to-video, image-to-video, and MiniMax-H3 multimodal reference)",
|
||||
zh: "MiniMax 海螺视频生成（文生视频、图生视频、MiniMax-H3 多模态参考生视频）",
|
||||
},
|
||||
version: "1.2.0",
|
||||
version: "1.3.0",
|
||||
author: { name: "QuantumNous" },
|
||||
channelTypes: [35],
|
||||
models: [
|
||||
"MiniMax-H3",
|
||||
"MiniMax-Hailuo-2.3",
|
||||
"MiniMax-Hailuo-2.3-Fast",
|
||||
"MiniMax-Hailuo-02",
|
||||
"T2V-01-Director",
|
||||
"T2V-01",
|
||||
"I2V-01-Director",
|
||||
"I2V-01-live",
|
||||
"I2V-01",
|
||||
"S2V-01",
|
||||
],
|
||||
models: [H3_MODEL].concat(V1_MODELS),
|
||||
fetchMode: "per_task",
|
||||
upstreams: ["vendor", "new_api"],
|
||||
// Fallback for a channel alias that resolves to more than one model: the
|
||||
// union of every family's fields and resolutions.
|
||||
usageSchema: Object.assign({}, H3_USAGE_SCHEMA, {
|
||||
@@ -134,6 +151,15 @@ export const meta = {
|
||||
examples: [{ label: "01-series 720P 6s", facts: { seconds: 6 } }],
|
||||
},
|
||||
],
|
||||
// MiniMax request and response bodies, for clients and for a downstream New
|
||||
// API gateway. The task id of a query is a path parameter on both versions,
|
||||
// where MiniMax /v1 reads it from the query string.
|
||||
routes: [
|
||||
{ method: "POST", path: "/hailuo/v1/video_generation", type: "submit", decode: "createTask", render: "taskCreated", models: V1_MODELS },
|
||||
{ method: "GET", path: "/hailuo/v1/query/video_generation/:task_id", type: "query", render: "taskStatus" },
|
||||
{ method: "POST", path: "/hailuo/v2/video_generation", type: "submit", decode: "createH3Task", render: "taskCreated", models: [H3_MODEL] },
|
||||
{ method: "GET", path: "/hailuo/v2/query/video_generation/:task_id", type: "query", render: "h3TaskStatus" },
|
||||
],
|
||||
protocols: [{ name: "openai_responses", supports: ["stream", "sync", "background"] }, "openai_video"],
|
||||
};
|
||||
|
||||
@@ -141,6 +167,29 @@ function trimmed(value) {
|
||||
return String(value || "").trim();
|
||||
}
|
||||
|
||||
function plainObject(value) {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
||||
}
|
||||
|
||||
// The members of source that hold a value.
|
||||
function definedFields(source, keys) {
|
||||
const fields = {};
|
||||
for (const key of keys) {
|
||||
if (source[key] !== undefined && source[key] !== null) fields[key] = source[key];
|
||||
}
|
||||
return fields;
|
||||
}
|
||||
|
||||
// On a New API channel the same plugin answers on another gateway, which
|
||||
// serves the MiniMax wire format only on the prefixed native routes.
|
||||
function viaGateway(ctx) {
|
||||
return Boolean(ctx.upstream && ctx.upstream.kind === "new_api");
|
||||
}
|
||||
|
||||
function apiRoot(ctx) {
|
||||
return ctx.baseUrl + (viaGateway(ctx) ? "/hailuo" : "");
|
||||
}
|
||||
|
||||
function isModernHailuo(model) {
|
||||
return model === "MiniMax-Hailuo-2.3" || model === "MiniMax-Hailuo-2.3-Fast" || model === "MiniMax-Hailuo-02";
|
||||
}
|
||||
@@ -186,7 +235,6 @@ function hasHailuoImage(req, hasInputReferenceFile) {
|
||||
);
|
||||
}
|
||||
|
||||
const H3_MODEL = "MiniMax-H3";
|
||||
const H3_MIN_DURATION = 4;
|
||||
const H3_MAX_DURATION = 15;
|
||||
const H3_DEFAULT_DURATION = 5;
|
||||
@@ -437,43 +485,106 @@ function responsesVideoText(ctx) {
|
||||
return '<video controls src="' + escaped + '"></video>';
|
||||
}
|
||||
|
||||
function nativeBody(ctx) {
|
||||
if (!ctx.body || ctx.body.kind !== "json") throw new Error("JSON body required");
|
||||
const body = plainObject(ctx.body.value);
|
||||
if (!body) throw new Error("request body must be an object");
|
||||
return body;
|
||||
}
|
||||
|
||||
// The decoders move the vendor fields to where the OpenAI-style requests carry
|
||||
// them, so buildSubmitRequest sends MiniMax the body the client sent.
|
||||
export const native = {
|
||||
createTask: function (ctx) {
|
||||
const body = nativeBody(ctx);
|
||||
const requestBody = Object.assign(definedFields(body, ["prompt", "duration", "resolution"]), {
|
||||
model: body.model,
|
||||
metadata: definedFields(body, V1_METADATA_FIELDS),
|
||||
});
|
||||
const hasImage = hasHailuoImage(requestBody, false);
|
||||
const duration = requestBody.duration === undefined ? undefined : Number(requestBody.duration);
|
||||
validateHailuoCombo(body.model, duration, outboundResolution(requestBody, body.model), hasImage);
|
||||
return { kind: "submit", model: body.model, action: hasImage ? "image_to_video" : "text_to_video", requestBody: requestBody };
|
||||
},
|
||||
createH3Task: function (ctx) {
|
||||
const body = nativeBody(ctx);
|
||||
if (!Array.isArray(body.content)) throw new Error("content must be an array");
|
||||
const content = validateH3Content(body.content);
|
||||
const requestBody = Object.assign(definedFields(body, ["duration", "resolution"]), {
|
||||
model: body.model,
|
||||
metadata: definedFields(body, ["content", "ratio"].concat(H3_METADATA_FIELDS)),
|
||||
});
|
||||
return { kind: "submit", model: body.model, action: h3HasVisualContent(content) ? "image_to_video" : "text_to_video", requestBody: requestBody };
|
||||
},
|
||||
// MiniMax answers /v1 with a base_resp envelope and /v2 with a bare task_id.
|
||||
taskCreated: function (ctx, task) {
|
||||
return Object.assign({}, plainObject(task.data) || {}, { task_id: task.task_id });
|
||||
},
|
||||
// A status route answers with the latest MiniMax snapshot. Before the first
|
||||
// poll that is the create response, which has no status, and the poller can
|
||||
// fail a task whose snapshot was still running; both take their status from
|
||||
// the task, so a downstream gateway reads the same state.
|
||||
taskStatus: function (ctx, task) {
|
||||
const data = plainObject(task.data) || {};
|
||||
const output = Object.assign({}, data, { task_id: task.task_id });
|
||||
if (task.status === "FAILURE") output.status = "Fail";
|
||||
else if (typeof data.status !== "string") output.status = task.status === "IN_PROGRESS" ? "Processing" : "Queueing";
|
||||
return output;
|
||||
},
|
||||
h3TaskStatus: function (ctx, task) {
|
||||
const data = plainObject(task.data) || {};
|
||||
if (h3APIError(data)) return data;
|
||||
const current = h3QueryTask(data);
|
||||
const output = Object.assign({}, current || {}, { id: task.task_id });
|
||||
if (task.status === "FAILURE" && output.status !== "failed" && output.status !== "cancelled") {
|
||||
output.status = "failed";
|
||||
output.error = { message: task.fail_reason || "task failed" };
|
||||
} else if (!current) {
|
||||
output.status = task.status === "IN_PROGRESS" ? "running" : "queued";
|
||||
}
|
||||
return Object.assign({}, data, { task: output });
|
||||
},
|
||||
// MiniMax reports a rejected call in base_resp on both versions.
|
||||
error: function (ctx, error) {
|
||||
return { base_resp: { status_code: error.httpStatus, status_msg: error.message } };
|
||||
},
|
||||
};
|
||||
|
||||
export function buildSubmitRequest(ctx) {
|
||||
const req = ctx.requestBody || {};
|
||||
const model = ctx.upstreamModel;
|
||||
const metadata = req.metadata || {};
|
||||
if (isH3(model)) {
|
||||
const content = h3Content(req);
|
||||
const h3Body = {
|
||||
model: model,
|
||||
content: content,
|
||||
resolution: h3Resolution(req),
|
||||
duration: h3Duration(req),
|
||||
ratio: h3Ratio(req, content),
|
||||
};
|
||||
["callback_url", "aigc_watermark"].forEach(function (key) {
|
||||
if (metadata[key] !== undefined && metadata[key] !== null) h3Body[key] = metadata[key];
|
||||
});
|
||||
const h3Body = Object.assign(
|
||||
{
|
||||
model: model,
|
||||
content: content,
|
||||
resolution: h3Resolution(req),
|
||||
duration: h3Duration(req),
|
||||
ratio: h3Ratio(req, content),
|
||||
},
|
||||
definedFields(metadata, H3_METADATA_FIELDS)
|
||||
);
|
||||
return {
|
||||
url: ctx.baseUrl + "/v2/video_generation",
|
||||
url: apiRoot(ctx) + "/v2/video_generation",
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", Authorization: "Bearer " + ctx.apiKey },
|
||||
body: h3Body,
|
||||
action: h3HasVisualContent(content) ? "image_to_video" : "text_to_video",
|
||||
};
|
||||
}
|
||||
const body = {
|
||||
model: model,
|
||||
prompt: req.prompt || undefined,
|
||||
duration: outboundDuration(req),
|
||||
resolution: outboundResolution(req, model),
|
||||
};
|
||||
["prompt_optimizer", "fast_pretreatment", "callback_url", "aigc_watermark", "first_frame_image", "last_frame_image", "subject_reference"].forEach(
|
||||
function (key) {
|
||||
if (metadata[key] !== undefined && metadata[key] !== null) body[key] = metadata[key];
|
||||
}
|
||||
const body = Object.assign(
|
||||
{
|
||||
model: model,
|
||||
prompt: req.prompt || undefined,
|
||||
duration: outboundDuration(req),
|
||||
resolution: outboundResolution(req, model),
|
||||
},
|
||||
definedFields(metadata, V1_METADATA_FIELDS)
|
||||
);
|
||||
return {
|
||||
url: ctx.baseUrl + "/v1/video_generation",
|
||||
url: apiRoot(ctx) + "/v1/video_generation",
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", Authorization: "Bearer " + ctx.apiKey },
|
||||
body: body,
|
||||
@@ -524,11 +635,12 @@ export function extractUsage(ctx) {
|
||||
export function buildQueryRequest(ctx) {
|
||||
// Polling carries no relay info; the host fills these identities from the
|
||||
// persisted task properties.
|
||||
const path = isH3(ctx.upstreamModel || ctx.model)
|
||||
? "/v2/query/video_generation/" + encodeURIComponent(ctx.taskId)
|
||||
: "/v1/query/video_generation?task_id=" + encodeURIComponent(ctx.taskId);
|
||||
const taskID = encodeURIComponent(ctx.taskId);
|
||||
let path = "/v1/query/video_generation?task_id=" + taskID;
|
||||
if (isH3(ctx.upstreamModel || ctx.model)) path = "/v2/query/video_generation/" + taskID;
|
||||
else if (viaGateway(ctx)) path = "/v1/query/video_generation/" + taskID;
|
||||
return {
|
||||
url: ctx.baseUrl + path,
|
||||
url: apiRoot(ctx) + path,
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json", Authorization: "Bearer " + ctx.apiKey },
|
||||
};
|
||||
@@ -600,11 +712,32 @@ export function buildContentRequest(ctx) {
|
||||
if (!url) throw new Error("artifact_not_found");
|
||||
return { url: url, method: ctx.clientRequest.method, credentialless: true };
|
||||
}
|
||||
return {
|
||||
url: ctx.baseUrl + "/v1/files/download?file_id=" + encodeURIComponent(fileID),
|
||||
method: ctx.clientRequest.method,
|
||||
headers: { Accept: "video/*", Authorization: "Bearer " + ctx.apiKey },
|
||||
};
|
||||
// The gateway resolves its MiniMax file itself and serves the video as the
|
||||
// artifact of its own task.
|
||||
if (viaGateway(ctx)) {
|
||||
return {
|
||||
url: ctx.baseUrl + "/v1/tasks/" + encodeURIComponent(ctx.upstreamTaskId) + "/artifacts/video/content",
|
||||
method: ctx.clientRequest.method,
|
||||
headers: { Authorization: "Bearer " + ctx.apiKey },
|
||||
};
|
||||
}
|
||||
// MiniMax serves v1 videos only through the signed download_url that
|
||||
// /v1/files/retrieve returns. The link is signed for GET, so a HEAD client
|
||||
// is served with GET as well; the host sends that client the headers only.
|
||||
const response = utils.fetch({
|
||||
url: ctx.baseUrl + "/v1/files/retrieve?file_id=" + encodeURIComponent(fileID),
|
||||
headers: { Accept: "application/json", Authorization: "Bearer " + ctx.apiKey },
|
||||
});
|
||||
if (!response.body || typeof response.body !== "object") {
|
||||
throw new Error("MiniMax file retrieve returned an unexpected response: HTTP " + response.status);
|
||||
}
|
||||
const base = response.body.base_resp || {};
|
||||
if (response.status !== 200 || base.status_code) {
|
||||
throw new Error("MiniMax file retrieve failed: " + (base.status_msg || "HTTP " + response.status));
|
||||
}
|
||||
const url = trimmed(response.body.file && response.body.file.download_url);
|
||||
if (!url) throw new Error("artifact_not_found");
|
||||
return { url: url, method: "GET", credentialless: true };
|
||||
}
|
||||
|
||||
export function extractUsageOnComplete(_task, _taskResult, body) {
|
||||
|
||||
@@ -7,11 +7,12 @@ export const meta = {
|
||||
en: "Volcengine Jimeng video generation (text-to-video, image-to-video, and first-and-last-frame)",
|
||||
zh: "火山引擎即梦视频生成（文生视频、图生视频、首尾帧）",
|
||||
},
|
||||
version: "1.0.2",
|
||||
version: "1.1.0",
|
||||
author: { name: "QuantumNous" },
|
||||
channelTypes: [51],
|
||||
models: ["jimeng_vgfm_t2v_l20"],
|
||||
fetchMode: "per_task",
|
||||
upstreams: ["vendor", "new_api"],
|
||||
usageSchema: {
|
||||
// Requested video duration in seconds. S2.0 Pro is fixed at 5; 3.0 req_keys allow 5 or 10.
|
||||
seconds: {
|
||||
@@ -90,6 +91,12 @@ function isRelay(apiKey) {
|
||||
return apiKey.startsWith("sk-");
|
||||
}
|
||||
|
||||
// The host signal is authoritative on New API channels; the sk- key prefix
|
||||
// stays as the heuristic for legacy type-51 channels pointed at a gateway.
|
||||
function viaGateway(ctx) {
|
||||
return !!(ctx.upstream && ctx.upstream.kind === "new_api") || isRelay(ctx.apiKey);
|
||||
}
|
||||
|
||||
function imageValues(body) {
|
||||
const images = [];
|
||||
if (Array.isArray(body.image_urls)) {
|
||||
@@ -153,13 +160,13 @@ function decodeNativeRequest(ctx) {
|
||||
};
|
||||
}
|
||||
|
||||
function endpoint(baseUrl, apiKey, action) {
|
||||
return baseUrl + (isRelay(apiKey) ? "/jimeng/" : "/") + "?Action=" + action + "&Version=2022-08-31";
|
||||
function endpoint(ctx, action) {
|
||||
return ctx.baseUrl + (viaGateway(ctx) ? "/jimeng/" : "/") + "?Action=" + action + "&Version=2022-08-31";
|
||||
}
|
||||
|
||||
function requestHeaders(ctx, method, url, bodyText) {
|
||||
const headers = { "Content-Type": "application/json", Accept: "application/json" };
|
||||
if (isRelay(ctx.apiKey)) {
|
||||
if (viaGateway(ctx)) {
|
||||
headers.Authorization = "Bearer " + ctx.apiKey;
|
||||
return headers;
|
||||
}
|
||||
@@ -337,7 +344,7 @@ export function buildSubmitRequest(ctx) {
|
||||
ordered.seed = body.seed;
|
||||
if (body.aspect_ratio) ordered.aspect_ratio = body.aspect_ratio;
|
||||
if (body.frames) ordered.frames = body.frames;
|
||||
const url = endpoint(ctx.baseUrl, ctx.apiKey, "CVSync2AsyncSubmitTask");
|
||||
const url = endpoint(ctx, "CVSync2AsyncSubmitTask");
|
||||
const bodyText = JSON.stringify(ordered);
|
||||
return {
|
||||
url: url,
|
||||
@@ -363,7 +370,7 @@ export function extractUsage(ctx) {
|
||||
|
||||
export function buildQueryRequest(ctx) {
|
||||
const body = JSON.stringify({ req_key: queryReqKey(ctx), task_id: ctx.taskId });
|
||||
const url = endpoint(ctx.baseUrl, ctx.apiKey, "CVSync2AsyncGetResult");
|
||||
const url = endpoint(ctx, "CVSync2AsyncGetResult");
|
||||
return { url: url, method: "POST", headers: requestHeaders(ctx, "POST", url, body), body: body };
|
||||
}
|
||||
|
||||
|
||||
@@ -7,11 +7,12 @@ export const meta = {
|
||||
en: "Kuaishou Kling video generation (text-to-video and image-to-video)",
|
||||
zh: "快手可灵视频生成（文生视频、图生视频）",
|
||||
},
|
||||
version: "1.0.2",
|
||||
version: "1.1.0",
|
||||
author: { name: "QuantumNous" },
|
||||
channelTypes: [50],
|
||||
models: ["kling-v1", "kling-v1-6", "kling-v2-master"],
|
||||
fetchMode: "per_task",
|
||||
upstreams: ["vendor", "new_api"],
|
||||
usageSchema: {
|
||||
// Kling final unit deduction (estimated at submit, actual on completion).
|
||||
units: {
|
||||
@@ -98,9 +99,15 @@ function isRelay(apiKey) {
|
||||
return apiKey.startsWith("sk-");
|
||||
}
|
||||
|
||||
function tokenFor(apiKey) {
|
||||
if (isRelay(apiKey)) return apiKey;
|
||||
const parts = apiKey.split("|");
|
||||
// The host signal is authoritative on New API channels; the sk- key prefix
|
||||
// stays as the heuristic for legacy type-50 channels pointed at a gateway.
|
||||
function viaGateway(ctx) {
|
||||
return !!(ctx.upstream && ctx.upstream.kind === "new_api") || isRelay(ctx.apiKey);
|
||||
}
|
||||
|
||||
function tokenFor(ctx) {
|
||||
if (viaGateway(ctx)) return ctx.apiKey;
|
||||
const parts = ctx.apiKey.split("|");
|
||||
if (parts.length !== 2) throw new Error("invalid api_key, required format is accessKey|secretKey");
|
||||
const now = utils.unixNow();
|
||||
return utils.jwtSignHS256({ iss: parts[0].trim(), exp: now + 1800, nbf: now - 5 }, parts[1].trim());
|
||||
@@ -110,8 +117,8 @@ function pathFor(action) {
|
||||
return action === "image_to_video" ? "/v1/videos/image2video" : "/v1/videos/text2video";
|
||||
}
|
||||
|
||||
function urlFor(baseUrl, apiKey, action) {
|
||||
return baseUrl + (isRelay(apiKey) ? "/kling" : "") + pathFor(action);
|
||||
function urlFor(ctx, action) {
|
||||
return ctx.baseUrl + (viaGateway(ctx) ? "/kling" : "") + pathFor(action);
|
||||
}
|
||||
|
||||
function aspectRatio(size) {
|
||||
@@ -248,9 +255,9 @@ export function buildSubmitRequest(ctx) {
|
||||
if (!body.prompt) delete body.prompt;
|
||||
if (!body.image) delete body.image;
|
||||
return {
|
||||
url: urlFor(ctx.baseUrl, ctx.apiKey, action),
|
||||
url: urlFor(ctx, action),
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", Authorization: "Bearer " + tokenFor(ctx.apiKey), "User-Agent": "kling-sdk/1.0" },
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", Authorization: "Bearer " + tokenFor(ctx), "User-Agent": "kling-sdk/1.0" },
|
||||
body: body,
|
||||
action: action,
|
||||
};
|
||||
@@ -274,9 +281,9 @@ export function extractUsage(ctx) {
|
||||
|
||||
export function buildQueryRequest(ctx) {
|
||||
return {
|
||||
url: urlFor(ctx.baseUrl, ctx.apiKey, ctx.action) + "/" + ctx.taskId,
|
||||
url: urlFor(ctx, ctx.action) + "/" + ctx.taskId,
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json", Authorization: "Bearer " + tokenFor(ctx.apiKey), "User-Agent": "kling-sdk/1.0" },
|
||||
headers: { Accept: "application/json", Authorization: "Bearer " + tokenFor(ctx), "User-Agent": "kling-sdk/1.0" },
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -7,11 +7,13 @@ export const meta = {
|
||||
en: "OpenAI Sora video generation (text-to-video, image-to-video, and remix)",
|
||||
zh: "OpenAI Sora 视频生成（文生视频、图生视频、remix）",
|
||||
},
|
||||
version: "1.0.3",
|
||||
version: "1.1.0",
|
||||
channelTypes: [55, 1], // OpenAI-type channels natively serve sora with the same wire format
|
||||
author: { name: "QuantumNous" },
|
||||
models: ["sora-2", "sora-2-pro"],
|
||||
fetchMode: "per_task",
|
||||
// A New API gateway serves /v1/videos as a host protocol, so no URL changes.
|
||||
upstreams: ["vendor", "new_api"],
|
||||
usageSchema: {
|
||||
// Requested video duration in seconds.
|
||||
seconds: {
|
||||
|
||||
@@ -9,11 +9,13 @@ export const meta = {
|
||||
en: "SunoAPI project music and lyrics generation",
|
||||
zh: "SunoAPI 项目 音乐与歌词生成",
|
||||
},
|
||||
version: "1.0.3",
|
||||
version: "1.1.0",
|
||||
author: { name: "QuantumNous" },
|
||||
channelTypes: [36],
|
||||
models: ["suno_music", "suno_lyrics"],
|
||||
fetchMode: "batch",
|
||||
// Vendor paths already equal the /suno native routes, so no URL changes.
|
||||
upstreams: ["vendor", "new_api"],
|
||||
usageSchema: {
|
||||
// Number of generated music or lyrics clips.
|
||||
clips: {
|
||||
|
||||
@@ -7,12 +7,13 @@ export const meta = {
|
||||
en: "Google Veo video generation on Vertex AI (text-to-video and image-to-video)",
|
||||
zh: "Google Veo 视频生成（文生视频、图生视频），Vertex AI 版本",
|
||||
},
|
||||
version: "1.0.2",
|
||||
version: "1.1.0",
|
||||
channelTypes: [41],
|
||||
author: { name: "QuantumNous" },
|
||||
models: ["veo-3.0-generate-001", "veo-3.0-fast-generate-001", "veo-3.1-generate-preview", "veo-3.1-fast-generate-preview"],
|
||||
fetchMode: "per_task",
|
||||
auth: { type: "oauth2_jwt" },
|
||||
upstreams: ["vendor", "new_api"],
|
||||
usageSchema: {
|
||||
// Requested video duration in seconds. Allowed values: 4, 6, 8.
|
||||
seconds: {
|
||||
@@ -38,12 +39,40 @@ export const meta = {
|
||||
{ label: "8s 1080p audio", facts: { seconds: 8, resolution: "1080p", generate_audio: true } },
|
||||
{ label: "8s 4k audio", facts: { seconds: 8, resolution: "4k", generate_audio: true } },
|
||||
],
|
||||
// Vertex AI request and response bodies, for clients and for a downstream New
|
||||
// API gateway. The model segment carries the method: predictLongRunning
|
||||
// submits and fetchPredictOperation polls. The gateway runs every request in
|
||||
// its own channel's project; the operation names it returns echo the path's
|
||||
// project and location and end with the gateway task id.
|
||||
routes: [
|
||||
{
|
||||
method: "POST",
|
||||
path: "/vertex-ai/v1/projects/:project/locations/:location/publishers/google/models/:model",
|
||||
type: "dynamic",
|
||||
decode: "modelMethod",
|
||||
render: "operation",
|
||||
},
|
||||
],
|
||||
protocols: [{ name: "openai_responses", supports: ["stream", "sync", "background"] }, "openai_video"],
|
||||
};
|
||||
|
||||
function trimmed(value) {
|
||||
return String(value || "").trim();
|
||||
}
|
||||
function plainObject(value) {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
||||
}
|
||||
// On a New API channel the same plugin answers on another gateway, which
|
||||
// serves the Vertex AI wire format on the prefixed native routes, takes its own
|
||||
// token as a Bearer header and runs the request in its own project.
|
||||
function viaGateway(ctx) {
|
||||
return Boolean(ctx.upstream && ctx.upstream.kind === "new_api");
|
||||
}
|
||||
function requestHeaders(ctx) {
|
||||
const headers = { "Content-Type": "application/json", Accept: "application/json", Authorization: ctx.authHeader };
|
||||
if (!viaGateway(ctx)) headers["x-goog-user-project"] = ctx.auth.projectId;
|
||||
return headers;
|
||||
}
|
||||
function responsesInput(req) {
|
||||
const texts = [],
|
||||
images = [];
|
||||
@@ -125,7 +154,7 @@ function apiBase(baseUrl, project, location) {
|
||||
return "https://" + (location === "global" ? "" : location + "-") + "aiplatform.googleapis.com/v1/projects/" + project + "/locations/" + location;
|
||||
}
|
||||
function modelURL(ctx, project, location, model, action) {
|
||||
return apiBase(ctx.baseUrl, project, location) + "/publishers/google/models/" + model + ":" + action;
|
||||
return apiBase(ctx.baseUrl + (viaGateway(ctx) ? "/vertex-ai" : ""), project, location) + "/publishers/google/models/" + model + ":" + action;
|
||||
}
|
||||
function decodeTaskId(value) {
|
||||
return utils.base64URLDecode(value);
|
||||
@@ -144,6 +173,71 @@ function dataVideo(response) {
|
||||
return "data:" + (String(enc).includes("/") ? enc : "video/" + enc) + ";base64," + data;
|
||||
}
|
||||
|
||||
export const native = {
|
||||
modelMethod: function (ctx) {
|
||||
const segment = String((ctx.params || {}).model || "");
|
||||
const separator = segment.lastIndexOf(":");
|
||||
const method = separator > 0 ? segment.slice(separator + 1) : "";
|
||||
if (!ctx.body || ctx.body.kind !== "json") throw new Error("JSON body required");
|
||||
const body = plainObject(ctx.body.value);
|
||||
if (!body) throw new Error("request body must be an object");
|
||||
if (method === "fetchPredictOperation") {
|
||||
const name = String(body.operationName || "");
|
||||
const marker = name.lastIndexOf("/operations/");
|
||||
if (marker < 0) throw new Error("operationName must name an operation");
|
||||
return { kind: "query", taskIds: [name.slice(marker + "/operations/".length)] };
|
||||
}
|
||||
if (method !== "predictLongRunning") throw new Error("the path must name models/{model}:predictLongRunning or :fetchPredictOperation");
|
||||
const model = segment.slice(0, separator);
|
||||
const instances = Array.isArray(body.instances) ? body.instances : [];
|
||||
const instance = plainObject(instances[0]);
|
||||
if (instances.length !== 1 || !instance) throw new Error("instances must hold exactly one instance");
|
||||
// buildSubmitRequest sends only the prompt and the first frame.
|
||||
for (const key of Object.keys(instance)) {
|
||||
if (key !== "prompt" && key !== "image") throw new Error("instances[0]." + key + " is not supported");
|
||||
}
|
||||
const metadata = Object.assign({}, plainObject(body.parameters) || {});
|
||||
const requestBody = { model: model, prompt: instance.prompt, metadata: metadata };
|
||||
if (metadata.durationSeconds !== undefined) {
|
||||
requestBody.duration = metadata.durationSeconds;
|
||||
delete metadata.durationSeconds;
|
||||
if ([4, 6, 8].indexOf(Number(requestBody.duration)) < 0) throw new Error("durationSeconds must be one of 4, 6, or 8");
|
||||
}
|
||||
if (metadata.resolution !== undefined && ["720p", "1080p", "4k"].indexOf(String(metadata.resolution).toLowerCase()) < 0)
|
||||
throw new Error("resolution must be one of 720p, 1080p, or 4k");
|
||||
if (instance.image !== undefined) {
|
||||
const image = plainObject(instance.image) || {};
|
||||
const mime = trimmed(image.mimeType).toLowerCase();
|
||||
if (!trimmed(image.bytesBase64Encoded)) throw new Error("instances[0].image must carry bytesBase64Encoded");
|
||||
if (mime !== "image/jpeg" && mime !== "image/png") throw new Error("input image must be image/jpeg or image/png");
|
||||
requestBody.images = ["data:" + mime + ";base64," + image.bytesBase64Encoded];
|
||||
}
|
||||
return { kind: "submit", model: model, action: requestBody.images ? "image_to_video" : "text_to_video", requestBody: requestBody };
|
||||
},
|
||||
// Submission renders one task and fetchPredictOperation a list of one. The
|
||||
// poller can fail a task whose operation was still running; that operation
|
||||
// is rendered done with the failure.
|
||||
operation: function (ctx, tasks) {
|
||||
const task = Array.isArray(tasks) ? tasks[0] : tasks;
|
||||
const params = ctx.params || {};
|
||||
const data = plainObject(task.data) || {};
|
||||
const named = /\/models\/([^/]+)\/operations\//.exec(String(data.name || ""));
|
||||
const model = named ? named[1] : String(params.model || "").split(":")[0];
|
||||
const output = Object.assign({}, data, {
|
||||
name: "projects/" + params.project + "/locations/" + params.location + "/publishers/google/models/" + model + "/operations/" + task.task_id,
|
||||
});
|
||||
if (task.status === "FAILURE" && !(plainObject(data.error) && data.error.message)) {
|
||||
output.done = true;
|
||||
output.error = { message: task.fail_reason || "task failed" };
|
||||
delete output.response;
|
||||
}
|
||||
return output;
|
||||
},
|
||||
error: function (ctx, error) {
|
||||
return { error: { code: error.httpStatus, message: error.message } };
|
||||
},
|
||||
};
|
||||
|
||||
export function buildSubmitRequest(ctx) {
|
||||
if (ctx.authError) throw new Error(ctx.authError);
|
||||
const req = ctx.requestBody || {},
|
||||
@@ -183,9 +277,10 @@ export function buildSubmitRequest(ctx) {
|
||||
instance.image = { bytesBase64Encoded: data, mimeType: mime };
|
||||
}
|
||||
return {
|
||||
url: modelURL(ctx, ctx.auth.projectId, region(ctx, model), model, "predictLongRunning"),
|
||||
// A gateway runs the request in its own project, so the path names none.
|
||||
url: modelURL(ctx, viaGateway(ctx) ? "-" : ctx.auth.projectId, region(ctx, model), model, "predictLongRunning"),
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", Authorization: ctx.authHeader, "x-goog-user-project": ctx.auth.projectId },
|
||||
headers: requestHeaders(ctx),
|
||||
body: { instances: [instance], parameters: metadata },
|
||||
action: image ? "image_to_video" : "text_to_video",
|
||||
};
|
||||
@@ -217,7 +312,7 @@ export function buildQueryRequest(ctx) {
|
||||
return {
|
||||
url: modelURL(ctx, project, location || "us-central1", model, "fetchPredictOperation"),
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", Authorization: ctx.authHeader, "x-goog-user-project": ctx.auth.projectId },
|
||||
headers: requestHeaders(ctx),
|
||||
body: { operationName: name },
|
||||
};
|
||||
}
|
||||
|
||||
@@ -7,11 +7,12 @@ export const meta = {
|
||||
en: "Shengshu Vidu video generation (text-to-video, image-to-video, first-and-last-frame, and reference-to-video)",
|
||||
zh: "生数 Vidu 视频生成（文生视频、图生视频、首尾帧、参考生视频）",
|
||||
},
|
||||
version: "1.0.2",
|
||||
version: "1.1.0",
|
||||
author: { name: "QuantumNous" },
|
||||
channelTypes: [52],
|
||||
models: ["viduq2", "viduq1", "vidu2.0", "vidu1.5"],
|
||||
fetchMode: "per_task",
|
||||
upstreams: ["vendor", "new_api"],
|
||||
usageSchema: {
|
||||
// estimated/actual Vidu credits
|
||||
credits: {
|
||||
@@ -46,15 +47,52 @@ export const meta = {
|
||||
{ label: "2.0 4s 720p", facts: { credits: 0, duration: 4, resolution: "720p" } },
|
||||
{ label: "2.0 8s 720p", facts: { credits: 0, duration: 8, resolution: "720p" } },
|
||||
],
|
||||
// Vidu request and response bodies, for clients and for a downstream New API
|
||||
// gateway.
|
||||
routes: [
|
||||
{ method: "POST", path: "/vidu/ent/v2/text2video", type: "submit", decode: "createTask", render: "taskCreated" },
|
||||
{ method: "POST", path: "/vidu/ent/v2/img2video", type: "submit", decode: "createTask", render: "taskCreated" },
|
||||
{ method: "POST", path: "/vidu/ent/v2/start-end2video", type: "submit", decode: "createTask", render: "taskCreated" },
|
||||
{ method: "POST", path: "/vidu/ent/v2/reference2video", type: "submit", decode: "createTask", render: "taskCreated" },
|
||||
{ method: "GET", path: "/vidu/ent/v2/tasks/:task_id/creations", type: "query", render: "taskStatus" },
|
||||
],
|
||||
protocols: [{ name: "openai_responses", supports: ["stream", "sync", "background"] }, "openai_video"],
|
||||
};
|
||||
|
||||
const RESOLUTIONS = ["360p", "540p", "720p", "1080p"];
|
||||
|
||||
// The submit route names the action; one or two reference images would
|
||||
// otherwise read as image-to-video or first-and-last-frame.
|
||||
const ROUTE_ACTIONS = {
|
||||
text2video: "text_to_video",
|
||||
img2video: "image_to_video",
|
||||
"start-end2video": "first_tail_to_video",
|
||||
reference2video: "reference_to_video",
|
||||
};
|
||||
|
||||
function trimmed(value) {
|
||||
return String(value || "").trim();
|
||||
}
|
||||
|
||||
function plainObject(value) {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
||||
}
|
||||
|
||||
// On a New API channel the same plugin answers on another gateway, which
|
||||
// serves the Vidu wire format on the prefixed native routes and takes its own
|
||||
// token as a Bearer header.
|
||||
function viaGateway(ctx) {
|
||||
return Boolean(ctx.upstream && ctx.upstream.kind === "new_api");
|
||||
}
|
||||
|
||||
function apiRoot(ctx) {
|
||||
return ctx.baseUrl + (viaGateway(ctx) ? "/vidu" : "") + "/ent/v2";
|
||||
}
|
||||
|
||||
function authorization(ctx) {
|
||||
return (viaGateway(ctx) ? "Bearer " : "Token ") + ctx.apiKey;
|
||||
}
|
||||
|
||||
function isQ2Model(model) {
|
||||
return String(model || "").indexOf("viduq2") === 0;
|
||||
}
|
||||
@@ -219,6 +257,45 @@ function pathFor(action) {
|
||||
return "/text2video";
|
||||
}
|
||||
|
||||
export const native = {
|
||||
createTask: function (ctx) {
|
||||
if (!ctx.body || ctx.body.kind !== "json") throw new Error("JSON body required");
|
||||
const body = plainObject(ctx.body.value);
|
||||
if (!body) throw new Error("request body must be an object");
|
||||
const model = trimmed(body.model);
|
||||
if (!model) throw new Error("model is required");
|
||||
const action = ROUTE_ACTIONS[ctx.path.slice(ctx.path.lastIndexOf("/") + 1)];
|
||||
// The other Vidu fields ride in metadata, which buildSubmitRequest
|
||||
// spreads back into the body.
|
||||
const metadata = Object.assign({}, body, { action: action });
|
||||
const requestBody = { model: model, metadata: metadata };
|
||||
delete metadata.model;
|
||||
for (const key of ["images", "prompt", "duration", "resolution"]) {
|
||||
delete metadata[key];
|
||||
if (body[key] !== undefined && body[key] !== null) requestBody[key] = body[key];
|
||||
}
|
||||
const duration = requestBody.duration === undefined ? undefined : Number(requestBody.duration);
|
||||
validateViduCombo(model, duration, outboundResolution(requestBody, model), hasViduImages(requestBody, false));
|
||||
return { kind: "submit", model: model, action: action, requestBody: requestBody };
|
||||
},
|
||||
taskCreated: function (ctx, task) {
|
||||
return Object.assign({}, plainObject(task.data) || {}, { task_id: task.task_id });
|
||||
},
|
||||
// The poller can fail a task whose snapshot was still processing; that
|
||||
// snapshot is rendered failed so a downstream gateway reads the same state.
|
||||
taskStatus: function (ctx, task) {
|
||||
const output = Object.assign({}, plainObject(task.data) || {}, { id: task.task_id });
|
||||
if (task.status === "FAILURE" && output.state !== "failed") {
|
||||
output.state = "failed";
|
||||
output.err_code = output.err_code || task.fail_reason || "task failed";
|
||||
}
|
||||
return output;
|
||||
},
|
||||
error: function (ctx, error) {
|
||||
return { code: error.code, message: error.message };
|
||||
},
|
||||
};
|
||||
|
||||
export function buildSubmitRequest(ctx) {
|
||||
const req = ctx.requestBody;
|
||||
const action = actionFor(req);
|
||||
@@ -226,25 +303,21 @@ export function buildSubmitRequest(ctx) {
|
||||
let model = ctx.upstreamModel || "viduq1";
|
||||
if (action === "reference_to_video" && model.includes("viduq2")) model = "viduq2";
|
||||
const images = Array.isArray(req.images) ? req.images.map(filePlaceholder) : null;
|
||||
const body = Object.assign(
|
||||
{
|
||||
model: model,
|
||||
images: images,
|
||||
prompt: req.prompt || null,
|
||||
duration: outboundDuration(req, model),
|
||||
resolution: outboundResolution(req, model),
|
||||
movement_amplitude: "auto",
|
||||
},
|
||||
metadata
|
||||
);
|
||||
// metadata carries the other Vidu fields and cannot replace the billed
|
||||
// model, duration and resolution.
|
||||
const body = Object.assign({ images: images, prompt: req.prompt || null, movement_amplitude: "auto" }, metadata, {
|
||||
model: model,
|
||||
duration: outboundDuration(req, model),
|
||||
resolution: outboundResolution(req, model),
|
||||
});
|
||||
delete body.action;
|
||||
if (!body.prompt) delete body.prompt;
|
||||
if (!body.bgm) delete body.bgm;
|
||||
if (Array.isArray(body.images)) body.images = body.images.map(filePlaceholder);
|
||||
return {
|
||||
url: ctx.baseUrl + "/ent/v2" + pathFor(action),
|
||||
url: apiRoot(ctx) + pathFor(action),
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", Authorization: "Token " + ctx.apiKey },
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", Authorization: authorization(ctx) },
|
||||
body: body,
|
||||
action: action,
|
||||
};
|
||||
@@ -265,9 +338,9 @@ export function extractUsage(ctx) {
|
||||
|
||||
export function buildQueryRequest(ctx) {
|
||||
return {
|
||||
url: ctx.baseUrl + "/ent/v2/tasks/" + ctx.taskId + "/creations",
|
||||
url: apiRoot(ctx) + "/tasks/" + ctx.taskId + "/creations",
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json", Authorization: "Token " + ctx.apiKey },
|
||||
headers: { Accept: "application/json", Authorization: authorization(ctx) },
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,783 @@
|
||||
// xAI bills each output second by model and resolution, each input image, and
|
||||
// on grok-imagine-video each input video second of an edit or extension
|
||||
// (docs.x.ai/developers/models/<model>).
|
||||
const VIDEO_MODELS = {
|
||||
// The only model that takes video input, so edits and extensions use it.
|
||||
"grok-imagine-video": { resolutions: ["480p", "720p"], videoInput: true },
|
||||
"grok-imagine-video-1.5": { resolutions: ["480p", "720p", "1080p"] },
|
||||
"grok-imagine-video-1.5-lite": { resolutions: ["480p", "720p", "1080p"] },
|
||||
};
|
||||
|
||||
const ASPECT_RATIOS = ["1:1", "16:9", "9:16", "4:3", "3:4", "3:2", "2:3"];
|
||||
const DEFAULT_RESOLUTION = "480p";
|
||||
const DEFAULT_DURATION = 8;
|
||||
const MAX_DURATION = 15;
|
||||
const DEFAULT_EXTENSION_SECONDS = 6;
|
||||
const MIN_EXTENSION_SECONDS = 2;
|
||||
const MAX_EXTENSION_SECONDS = 10;
|
||||
// Edits accept at most 8.7 seconds of input and return the same length.
|
||||
const MAX_EDIT_SECONDS = 9;
|
||||
// Extensions accept 2 to 15 seconds of input and return input plus extension.
|
||||
const MAX_EXTENSION_INPUT_SECONDS = 15;
|
||||
const MAX_REFERENCE_IMAGES = 7;
|
||||
const MAX_KEYFRAMES = 4;
|
||||
const MAX_REFERENCE_AUDIOS = 3;
|
||||
// Edits and extensions keep the input video's resolution, capped at 720p,
|
||||
// which neither the request nor the result reports: the cap is reserved and
|
||||
// the output file's MP4 header settles the resolution.
|
||||
const EDIT_RESOLUTION_CAP = "720p";
|
||||
|
||||
const SECONDS_FIELD = {
|
||||
type: "number",
|
||||
unit: "second",
|
||||
description: { en: "Video generation unit price", zh: "视频生成单价" },
|
||||
};
|
||||
const INPUT_IMAGES_FIELD = {
|
||||
type: "number",
|
||||
unit: "count",
|
||||
unitLabel: { en: "image", zh: "张", "zh-TW": "張" },
|
||||
description: { en: "Input image unit price", zh: "输入图片单价" },
|
||||
};
|
||||
const INPUT_VIDEO_SECONDS_FIELD = {
|
||||
type: "number",
|
||||
unit: "second",
|
||||
description: { en: "Input video unit price", zh: "输入视频单价" },
|
||||
};
|
||||
// xAI still bills a request its moderation blocks ("Usage Guidelines
|
||||
// Violation Fee", docs.x.ai/developers/pricing), so such a task settles as
|
||||
// charged instead of refunded; this fact lets its price differ from a
|
||||
// delivered video.
|
||||
const MODERATED_FIELD = {
|
||||
type: "boolean",
|
||||
description: { en: "Blocked by content moderation", zh: "被内容审核拦截" },
|
||||
};
|
||||
const RESOLUTION_DESCRIPTION = { en: "Output video resolution", zh: "输出视频分辨率" };
|
||||
|
||||
const GROK_IMAGINE_VIDEO_USAGE_SCHEMA = {
|
||||
seconds: SECONDS_FIELD,
|
||||
resolution: { enum: VIDEO_MODELS["grok-imagine-video"].resolutions, description: RESOLUTION_DESCRIPTION },
|
||||
input_images: INPUT_IMAGES_FIELD,
|
||||
input_video_seconds: INPUT_VIDEO_SECONDS_FIELD,
|
||||
moderated: MODERATED_FIELD,
|
||||
};
|
||||
|
||||
const GROK_IMAGINE_VIDEO_1_5_USAGE_SCHEMA = {
|
||||
seconds: SECONDS_FIELD,
|
||||
resolution: { enum: VIDEO_MODELS["grok-imagine-video-1.5"].resolutions, description: RESOLUTION_DESCRIPTION },
|
||||
input_images: INPUT_IMAGES_FIELD,
|
||||
moderated: MODERATED_FIELD,
|
||||
};
|
||||
|
||||
export const meta = {
|
||||
apiVersion: 1,
|
||||
key: "xai",
|
||||
name: "xAI",
|
||||
icon: "XAI",
|
||||
description: {
|
||||
en: "xAI Grok Imagine video generation (text-to-video, image-to-video, reference-to-video, video editing, and video extension)",
|
||||
zh: "xAI Grok Imagine 视频生成（文生视频、图生视频、参考生视频、视频编辑、视频延长）",
|
||||
},
|
||||
version: "1.1.0",
|
||||
author: { name: "QuantumNous" },
|
||||
channelTypes: [48], // xAI-type channels serve the video API with the same key
|
||||
baseUrl: "https://api.x.ai",
|
||||
// Result videos, whose headers settle the resolution of edits and extensions.
|
||||
allowedHosts: ["vidgen.x.ai"],
|
||||
models: Object.keys(VIDEO_MODELS),
|
||||
fetchMode: "per_task",
|
||||
upstreams: ["vendor", "new_api"],
|
||||
// Fallback for a channel alias that resolves to more than one model: the
|
||||
// union of every model's fields and resolutions.
|
||||
usageSchema: Object.assign({}, GROK_IMAGINE_VIDEO_USAGE_SCHEMA, {
|
||||
resolution: { enum: ["480p", "720p", "1080p"], description: RESOLUTION_DESCRIPTION },
|
||||
}),
|
||||
usageExamples: [
|
||||
{ label: "1.5 480p 8s", facts: { seconds: 8, resolution: "480p", input_images: 0, input_video_seconds: 0, moderated: false } },
|
||||
{ label: "1.5 1080p 8s · image", facts: { seconds: 8, resolution: "1080p", input_images: 1, input_video_seconds: 0, moderated: false } },
|
||||
{ label: "edit 8s · 720p", facts: { seconds: 8, resolution: "720p", input_images: 0, input_video_seconds: 8, moderated: false } },
|
||||
{ label: "480p 8s · blocked", facts: { seconds: 8, resolution: "480p", input_images: 0, input_video_seconds: 0, moderated: true } },
|
||||
],
|
||||
usageProfiles: [
|
||||
{
|
||||
models: ["grok-imagine-video"],
|
||||
schema: GROK_IMAGINE_VIDEO_USAGE_SCHEMA,
|
||||
examples: [
|
||||
{ label: "480p 8s", facts: { seconds: 8, resolution: "480p", input_images: 0, input_video_seconds: 0, moderated: false } },
|
||||
{ label: "720p 8s · image", facts: { seconds: 8, resolution: "720p", input_images: 1, input_video_seconds: 0, moderated: false } },
|
||||
{ label: "edit 8s · 720p", facts: { seconds: 8, resolution: "720p", input_images: 0, input_video_seconds: 8, moderated: false } },
|
||||
{ label: "extend 8s + 6s · 480p", facts: { seconds: 6, resolution: "480p", input_images: 0, input_video_seconds: 8, moderated: false } },
|
||||
{ label: "480p 8s · blocked", facts: { seconds: 8, resolution: "480p", input_images: 0, input_video_seconds: 0, moderated: true } },
|
||||
],
|
||||
},
|
||||
{
|
||||
models: ["grok-imagine-video-1.5", "grok-imagine-video-1.5-lite"],
|
||||
schema: GROK_IMAGINE_VIDEO_1_5_USAGE_SCHEMA,
|
||||
examples: [
|
||||
{ label: "480p 8s", facts: { seconds: 8, resolution: "480p", input_images: 0, moderated: false } },
|
||||
{ label: "720p 8s · image", facts: { seconds: 8, resolution: "720p", input_images: 1, moderated: false } },
|
||||
{ label: "1080p 15s", facts: { seconds: 15, resolution: "1080p", input_images: 0, moderated: false } },
|
||||
{ label: "480p 8s · blocked", facts: { seconds: 8, resolution: "480p", input_images: 0, moderated: true } },
|
||||
],
|
||||
},
|
||||
],
|
||||
routes: [
|
||||
{ method: "POST", path: "/xai/v1/videos/generations", type: "submit", decode: "createGeneration", render: "videoCreated" },
|
||||
{ method: "POST", path: "/xai/v1/videos/edits", type: "submit", decode: "createEdit", render: "videoCreated", models: ["grok-imagine-video"] },
|
||||
{ method: "POST", path: "/xai/v1/videos/extensions", type: "submit", decode: "createExtension", render: "videoCreated", models: ["grok-imagine-video"] },
|
||||
{ method: "GET", path: "/xai/v1/videos/:request_id", type: "query", taskIdParam: "request_id", render: "videoStatus" },
|
||||
],
|
||||
protocols: [{ name: "openai_responses", supports: ["stream", "sync", "background"] }, "openai_video"],
|
||||
};
|
||||
|
||||
function trimmed(value) {
|
||||
return String(value || "").trim();
|
||||
}
|
||||
|
||||
function plainObject(value) {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : null;
|
||||
}
|
||||
|
||||
// Another New API gateway serves the xAI wire format only on this plugin's
|
||||
// prefixed native routes.
|
||||
function apiRoot(ctx) {
|
||||
return ctx.baseUrl + (ctx.upstream && ctx.upstream.kind === "new_api" ? "/xai" : "");
|
||||
}
|
||||
|
||||
function videoModel(identity) {
|
||||
const names = [identity.upstreamModel, identity.model].map(trimmed).filter(Boolean);
|
||||
for (const name of names) {
|
||||
if (VIDEO_MODELS[name]) return name;
|
||||
}
|
||||
throw new Error((names[0] || "model") + " is not a supported xAI video model");
|
||||
}
|
||||
|
||||
function integerField(value, name, minimum, maximum) {
|
||||
const number = typeof value === "string" && value.trim() !== "" ? Number(value) : value;
|
||||
if (typeof number !== "number" || !Number.isInteger(number) || number < minimum || number > maximum)
|
||||
throw new Error(name + " must be an integer between " + minimum + " and " + maximum);
|
||||
return number;
|
||||
}
|
||||
|
||||
// Media inputs are URLs, base64 data URLs, or host file placeholders. A
|
||||
// file_id would name a file in the gateway's own xAI account, which callers
|
||||
// must not reach.
|
||||
function mediaInput(value, name) {
|
||||
const object = plainObject(value);
|
||||
if (object && object.file_id !== undefined) throw new Error(name + " must be a URL; file_id is not supported");
|
||||
const url = object ? (object.url !== undefined ? object.url : object.image_url) : value;
|
||||
if (plainObject(url) && url.__fileRef) return { url: url };
|
||||
if (typeof url === "string" && url.trim()) return { url: url.trim() };
|
||||
throw new Error(name + " must be a URL or {url}");
|
||||
}
|
||||
|
||||
function promptField(source, required) {
|
||||
if (source.prompt !== undefined && source.prompt !== null && typeof source.prompt !== "string") throw new Error("prompt must be a string");
|
||||
const prompt = trimmed(source.prompt);
|
||||
if (required && !prompt) throw new Error("prompt is required");
|
||||
return prompt;
|
||||
}
|
||||
|
||||
// Fields every operation accepts. storage_options is never forwarded: it
|
||||
// stores the output in the gateway's xAI account.
|
||||
function commonFields(source, body) {
|
||||
if (source.user !== undefined && source.user !== null) {
|
||||
if (typeof source.user !== "string") throw new Error("user must be a string");
|
||||
body.user = source.user;
|
||||
}
|
||||
if (source.output !== undefined && source.output !== null) {
|
||||
const output = plainObject(source.output);
|
||||
if (!output || typeof output.upload_url !== "string" || !output.upload_url.trim()) throw new Error("output.upload_url must be a URL");
|
||||
body.output = { upload_url: output.upload_url.trim() };
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
// Validates a /v1/videos/generations request and keeps only documented
|
||||
// fields, so an unlisted field cannot carry a billable quantity past these
|
||||
// checks. Duration and resolution are always sent so the reserved charge
|
||||
// matches what xAI renders even if its defaults change.
|
||||
function generationRequest(source, identity) {
|
||||
const model = videoModel(identity);
|
||||
const image = source.image === undefined || source.image === null ? null : mediaInput(source.image, "image");
|
||||
if (source.reference_images !== undefined && source.reference_images !== null && !Array.isArray(source.reference_images))
|
||||
throw new Error("reference_images must be an array");
|
||||
if ((source.reference_images || []).length > MAX_REFERENCE_IMAGES) throw new Error("reference_images accepts at most " + MAX_REFERENCE_IMAGES + " items");
|
||||
const referenceImages = (source.reference_images || []).map(function (item, index) {
|
||||
return mediaInput(item, "reference_images[" + index + "]");
|
||||
});
|
||||
const lastFrame = source.last_frame === undefined || source.last_frame === null ? null : mediaInput(source.last_frame, "last_frame");
|
||||
if (source.keyframes !== undefined && source.keyframes !== null && !Array.isArray(source.keyframes)) throw new Error("keyframes must be an array");
|
||||
const keyframes = (source.keyframes || []).map(function (keyframe, index) {
|
||||
const object = plainObject(keyframe);
|
||||
const timestamp = object ? Number(object.timestamp_s) : NaN;
|
||||
if (!Number.isFinite(timestamp) || timestamp <= 0) throw new Error("keyframes[" + index + "].timestamp_s must be a positive number");
|
||||
return { image: mediaInput(object.image, "keyframes[" + index + "].image"), timestamp_s: timestamp };
|
||||
});
|
||||
if (keyframes.length > MAX_KEYFRAMES) throw new Error("keyframes accepts at most " + MAX_KEYFRAMES + " items");
|
||||
if (source.reference_audios !== undefined && source.reference_audios !== null && !Array.isArray(source.reference_audios))
|
||||
throw new Error("reference_audios must be an array");
|
||||
const referenceAudios = (source.reference_audios || []).map(function (audio, index) {
|
||||
const object = plainObject(audio) || {};
|
||||
if (typeof object.voice_id === "string" && object.voice_id.trim()) return { voice_id: object.voice_id.trim() };
|
||||
if (typeof object.url === "string" && object.url.trim()) return { url: object.url.trim() };
|
||||
throw new Error("reference_audios[" + index + "] must be {voice_id} or {url}");
|
||||
});
|
||||
if (referenceAudios.length > MAX_REFERENCE_AUDIOS) throw new Error("reference_audios accepts at most " + MAX_REFERENCE_AUDIOS + " items");
|
||||
|
||||
let action = "text_to_video";
|
||||
if (referenceImages.length || referenceAudios.length) action = "reference_to_video";
|
||||
else if (image || lastFrame || keyframes.length) action = "image_to_video";
|
||||
const prompt = promptField(source, action !== "image_to_video");
|
||||
|
||||
const resolution = source.resolution === undefined || source.resolution === null ? DEFAULT_RESOLUTION : trimmed(source.resolution).toLowerCase();
|
||||
const resolutions = VIDEO_MODELS[model].resolutions;
|
||||
if (!resolutions.includes(resolution)) throw new Error(model + " resolution must be one of " + resolutions.join(", "));
|
||||
if (action === "reference_to_video" && resolution === "1080p") throw new Error("reference-to-video supports resolution 480p or 720p");
|
||||
const duration = source.duration === undefined || source.duration === null ? DEFAULT_DURATION : integerField(source.duration, "duration", 1, MAX_DURATION);
|
||||
|
||||
const body = { duration: duration, resolution: resolution };
|
||||
if (prompt) body.prompt = prompt;
|
||||
if (source.aspect_ratio !== undefined && source.aspect_ratio !== null) {
|
||||
if (!ASPECT_RATIOS.includes(source.aspect_ratio)) throw new Error("aspect_ratio must be one of " + ASPECT_RATIOS.join(", "));
|
||||
body.aspect_ratio = source.aspect_ratio;
|
||||
}
|
||||
if (image) body.image = image;
|
||||
if (referenceImages.length) body.reference_images = referenceImages;
|
||||
if (lastFrame) body.last_frame = lastFrame;
|
||||
if (keyframes.length) body.keyframes = keyframes;
|
||||
if (referenceAudios.length) body.reference_audios = referenceAudios;
|
||||
if (source.generate_audio !== undefined && source.generate_audio !== null) {
|
||||
if (typeof source.generate_audio !== "boolean") throw new Error("generate_audio must be a boolean");
|
||||
body.generate_audio = source.generate_audio;
|
||||
}
|
||||
return { action: action, body: commonFields(source, body) };
|
||||
}
|
||||
|
||||
// Edits and extensions take an input video; the output keeps its resolution
|
||||
// and aspect ratio, so those fields are not forwarded.
|
||||
function videoInputRequest(action, source, identity) {
|
||||
const model = videoModel(identity);
|
||||
if (!VIDEO_MODELS[model].videoInput) throw new Error("video editing and extension require grok-imagine-video");
|
||||
const body = { prompt: promptField(source, true), video: mediaInput(source.video, "video") };
|
||||
if (action === "video_extension") {
|
||||
body.duration =
|
||||
source.duration === undefined || source.duration === null
|
||||
? DEFAULT_EXTENSION_SECONDS
|
||||
: integerField(source.duration, "duration", MIN_EXTENSION_SECONDS, MAX_EXTENSION_SECONDS);
|
||||
}
|
||||
return { action: action, body: commonFields(source, body) };
|
||||
}
|
||||
|
||||
function videoRequest(action, source, identity) {
|
||||
if (action === "video_edit" || action === "video_extension") return videoInputRequest(action, source, identity);
|
||||
return generationRequest(generationSource(source), identity);
|
||||
}
|
||||
|
||||
// OpenAI-style WIDTHxHEIGHT sizes select the nearest xAI aspect ratio and the
|
||||
// resolution of the shorter side.
|
||||
function sizeOptions(size) {
|
||||
const text = trimmed(size).toLowerCase();
|
||||
if (["480p", "720p", "1080p"].includes(text)) return { resolution: text };
|
||||
const parts = text.replace("*", "x").split("x");
|
||||
const width = Number(parts[0]);
|
||||
const height = Number(parts[1]);
|
||||
if (parts.length !== 2 || !(width > 0) || !(height > 0)) throw new Error("size must be WIDTHxHEIGHT or one of 480p, 720p, 1080p");
|
||||
let aspectRatio = ASPECT_RATIOS[0];
|
||||
let nearest = Infinity;
|
||||
for (const ratio of ASPECT_RATIOS) {
|
||||
const sides = ratio.split(":").map(Number);
|
||||
const distance = Math.abs(Math.log(width / height) - Math.log(sides[0] / sides[1]));
|
||||
if (distance < nearest) {
|
||||
nearest = distance;
|
||||
aspectRatio = ratio;
|
||||
}
|
||||
}
|
||||
const shorter = Math.min(width, height);
|
||||
return { aspect_ratio: aspectRatio, resolution: shorter >= 1080 ? "1080p" : shorter >= 720 ? "720p" : "480p" };
|
||||
}
|
||||
|
||||
// Normalizes every request shape that reaches the driver: the canonical xAI
|
||||
// body, OpenAI video fields (seconds, size, input_reference), and the legacy
|
||||
// task shape (images, metadata) that host routes without a plugin decoder
|
||||
// pass through. Explicit xAI fields win over metadata and size.
|
||||
function generationSource(req) {
|
||||
const source = Object.assign(
|
||||
{},
|
||||
plainObject(req.metadata) || {},
|
||||
req.size === undefined || req.size === null || req.size === "" ? {} : sizeOptions(req.size),
|
||||
);
|
||||
for (const key of Object.keys(req)) {
|
||||
if (req[key] !== undefined && req[key] !== null && req[key] !== "") source[key] = req[key];
|
||||
}
|
||||
if (source.duration === undefined && source.seconds !== undefined) source.duration = source.seconds;
|
||||
if (source.image === undefined && source.input_reference !== undefined) source.image = source.input_reference;
|
||||
const images = Array.isArray(req.images) ? req.images : [];
|
||||
if (source.image === undefined && images.length === 1) source.image = images[0];
|
||||
if (source.reference_images === undefined && images.length > 1) source.reference_images = images;
|
||||
return source;
|
||||
}
|
||||
|
||||
// A moderated output still reports done, with respect_moderation false and no URL.
|
||||
function filteredOutput(data) {
|
||||
const video = plainObject(data) && data.status === "done" ? plainObject(data.video) : null;
|
||||
return Boolean(video && video.respect_moderation === false);
|
||||
}
|
||||
|
||||
// xAI reports the amount it billed on every video result. A failed result it
|
||||
// still billed is charged rather than refunded.
|
||||
function chargedFailure(data) {
|
||||
const usage = plainObject(data) && data.status === "failed" ? plainObject(data.usage) : null;
|
||||
return Boolean(usage && Number(usage.cost_in_usd_ticks) > 0);
|
||||
}
|
||||
|
||||
function moderationNotice(data) {
|
||||
if (filteredOutput(data)) return "The generated video was blocked by content moderation.";
|
||||
if (!chargedFailure(data)) return "";
|
||||
const error = plainObject(data.error);
|
||||
return error && trimmed(error.message) ? trimmed(error.message) : "The request was blocked by content moderation.";
|
||||
}
|
||||
|
||||
function videoURL(data) {
|
||||
const video = plainObject(data) && data.status === "done" ? plainObject(data.video) : null;
|
||||
return video && video.respect_moderation !== false ? trimmed(video.url) : "";
|
||||
}
|
||||
|
||||
function failureReason(data) {
|
||||
const error = plainObject(data && data.error);
|
||||
return error && trimmed(error.message) ? trimmed(error.message) : "video generation failed";
|
||||
}
|
||||
|
||||
function responsesInput(req) {
|
||||
const texts = [],
|
||||
images = [];
|
||||
const input = req.input;
|
||||
if (typeof input === "string") texts.push(input);
|
||||
else if (Array.isArray(input)) {
|
||||
for (const item of input) {
|
||||
if (typeof item === "string") {
|
||||
texts.push(item);
|
||||
continue;
|
||||
}
|
||||
if (!item || typeof item !== "object" || Array.isArray(item)) continue;
|
||||
const content = item.content === undefined ? [item] : Array.isArray(item.content) ? item.content : [item.content];
|
||||
for (const part of content) {
|
||||
if (typeof part === "string") {
|
||||
texts.push(part);
|
||||
continue;
|
||||
}
|
||||
if (!part || typeof part !== "object" || Array.isArray(part)) continue;
|
||||
if (["input_text", "text"].includes(part.type) && typeof part.text === "string") texts.push(part.text);
|
||||
if (["input_image", "image_url"].includes(part.type)) {
|
||||
let image = part.image_url;
|
||||
if (image && typeof image === "object") image = image.url;
|
||||
if (trimmed(image)) images.push(trimmed(image));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
prompt: texts
|
||||
.filter(function (text) {
|
||||
return trimmed(text);
|
||||
})
|
||||
.join("\n"),
|
||||
images: images,
|
||||
};
|
||||
}
|
||||
|
||||
function responsesVideoText(ctx, task) {
|
||||
const notice = moderationNotice(task && task.data);
|
||||
if (notice) return notice;
|
||||
const artifact = ctx && ctx.artifacts && ctx.artifacts.video;
|
||||
const url = trimmed(artifact && artifact.url);
|
||||
if (!url) throw new Error("video artifact is unavailable");
|
||||
const escaped = url.replace(/&/g, "&").replace(/"/g, """).replace(/</g, "<").replace(/>/g, ">");
|
||||
return '<video controls src="' + escaped + '"></video>';
|
||||
}
|
||||
|
||||
function nativeRequest(ctx, action) {
|
||||
if (!ctx.body || ctx.body.kind !== "json") throw new Error("JSON body required");
|
||||
const body = plainObject(ctx.body.value);
|
||||
if (!body) throw new Error("request body must be an object");
|
||||
const model = typeof body.model === "string" ? body.model.trim() : "";
|
||||
if (!model) throw new Error("model is required");
|
||||
const request = videoRequest(action, body, { model: model });
|
||||
return { kind: "submit", model: model, action: request.action, requestBody: Object.assign(request.body, { model: model }) };
|
||||
}
|
||||
|
||||
export const native = {
|
||||
createGeneration: function (ctx) {
|
||||
return nativeRequest(ctx, "generation");
|
||||
},
|
||||
createEdit: function (ctx) {
|
||||
return nativeRequest(ctx, "video_edit");
|
||||
},
|
||||
createExtension: function (ctx) {
|
||||
return nativeRequest(ctx, "video_extension");
|
||||
},
|
||||
videoCreated: function (ctx, task) {
|
||||
return { request_id: task.task_id };
|
||||
},
|
||||
videoStatus: function (ctx, task) {
|
||||
const data = plainObject(task.data);
|
||||
// The poller can fail a task whose last snapshot was still pending.
|
||||
if (data && typeof data.status === "string" && !(task.status === "FAILURE" && data.status === "pending")) {
|
||||
const output = Object.assign({}, data);
|
||||
delete output.request_id;
|
||||
// The gateway's upstream cost is not the caller's price.
|
||||
delete output.usage;
|
||||
// A billed failure settled as charged. Report it in xAI's moderated
|
||||
// output shape so callers, including a downstream gateway running this
|
||||
// plugin, see a charged result without the upstream cost.
|
||||
if (task.status === "SUCCESS" && chargedFailure(data)) {
|
||||
output.status = "done";
|
||||
output.progress = 100;
|
||||
output.video = { respect_moderation: false };
|
||||
}
|
||||
return output;
|
||||
}
|
||||
if (task.status === "FAILURE") return { status: "failed", error: { code: "internal_error", message: task.fail_reason || "video generation failed" } };
|
||||
const progress = Number(String(task.progress || "0").replace("%", ""));
|
||||
return { status: "pending", progress: Number.isFinite(progress) ? Math.min(Math.max(progress, 0), 99) : 0 };
|
||||
},
|
||||
error: function (ctx, error) {
|
||||
return { error: { code: error.code, message: error.message } };
|
||||
},
|
||||
};
|
||||
|
||||
export function buildSubmitRequest(ctx) {
|
||||
const request = videoRequest(ctx.action, ctx.requestBody || {}, ctx);
|
||||
const paths = { video_edit: "/v1/videos/edits", video_extension: "/v1/videos/extensions" };
|
||||
return {
|
||||
url: apiRoot(ctx) + (paths[request.action] || "/v1/videos/generations"),
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json", Authorization: "Bearer " + ctx.apiKey },
|
||||
body: Object.assign({}, request.body, { model: ctx.upstreamModel || ctx.model }),
|
||||
action: request.action,
|
||||
};
|
||||
}
|
||||
|
||||
export function parseSubmitResponse(ctx, resp) {
|
||||
const body = plainObject(resp.body);
|
||||
const taskId = body && typeof body.request_id === "string" ? body.request_id.trim() : "";
|
||||
if (!taskId) throw new Error("xAI did not return a request_id");
|
||||
const result = { taskId: taskId, taskData: body };
|
||||
// The result reports only the total length, so completion needs the
|
||||
// extension length to derive the input video seconds.
|
||||
if (ctx.action === "video_extension") result.state = { extension_seconds: (ctx.requestBody || {}).duration || DEFAULT_EXTENSION_SECONDS };
|
||||
return result;
|
||||
}
|
||||
|
||||
export function extractUsage(ctx) {
|
||||
const request = videoRequest(ctx.action, ctx.requestBody || {}, ctx);
|
||||
const body = request.body;
|
||||
const videoInput = VIDEO_MODELS[videoModel(ctx)].videoInput;
|
||||
let facts;
|
||||
if (request.action === "video_edit") {
|
||||
// Input URLs expose neither duration nor resolution; reserve the
|
||||
// documented maximum and the resolution cap, and settle on the result.
|
||||
facts = { seconds: MAX_EDIT_SECONDS, resolution: EDIT_RESOLUTION_CAP, input_images: 0, input_video_seconds: MAX_EDIT_SECONDS };
|
||||
} else if (request.action === "video_extension") {
|
||||
facts = { seconds: body.duration, resolution: EDIT_RESOLUTION_CAP, input_images: 0, input_video_seconds: MAX_EXTENSION_INPUT_SECONDS };
|
||||
} else {
|
||||
const images = (body.image ? 1 : 0) + (body.reference_images || []).length + (body.last_frame ? 1 : 0) + (body.keyframes || []).length;
|
||||
facts = { seconds: body.duration, resolution: body.resolution, input_images: images, input_video_seconds: 0 };
|
||||
}
|
||||
// Legacy per-call prices scale with the output length only.
|
||||
if (ctx.usagePurpose === "billing_ratios") return { seconds: facts.seconds };
|
||||
if (!videoInput) delete facts.input_video_seconds;
|
||||
facts.moderated = false;
|
||||
return facts;
|
||||
}
|
||||
|
||||
export function buildQueryRequest(ctx) {
|
||||
return {
|
||||
url: apiRoot(ctx) + "/v1/videos/" + encodeURIComponent(ctx.taskId),
|
||||
method: "GET",
|
||||
headers: { Accept: "application/json", Authorization: "Bearer " + ctx.apiKey },
|
||||
};
|
||||
}
|
||||
|
||||
export function parseTaskResult(ctx, body, response) {
|
||||
const data = plainObject(body);
|
||||
// xAI answers 202 while the video renders.
|
||||
if (!data && response && response.status === 202) return { status: "IN_PROGRESS" };
|
||||
const status = data ? data.status : undefined;
|
||||
if (status === "pending") {
|
||||
const result = { status: "IN_PROGRESS" };
|
||||
const progress = Number(data.progress);
|
||||
if (progress > 0 && progress < 100) result.progress = progress + "%";
|
||||
return result;
|
||||
}
|
||||
// A moderated output is done without a video. xAI still bills it, and only
|
||||
// a SUCCESS task keeps its charge.
|
||||
if (status === "done") {
|
||||
const result = { status: "SUCCESS" };
|
||||
// An edit or extension is billed by the resolution of its output, which
|
||||
// only the file reveals; an unreadable file keeps the reserved cap.
|
||||
if ((ctx.action === "video_edit" || ctx.action === "video_extension") && videoURL(data)) {
|
||||
const resolution = outputResolution(videoURL(data));
|
||||
if (resolution) result.state = Object.assign({}, plainObject(ctx.state) || {}, { output_resolution: resolution });
|
||||
}
|
||||
return result;
|
||||
}
|
||||
if (status === "failed") return chargedFailure(data) ? { status: "SUCCESS" } : { status: "FAILURE", reason: failureReason(data) };
|
||||
if (status === "expired") return { status: "FAILURE", reason: "the video expired before it was retrieved" };
|
||||
return { status: "UNKNOWN", reason: "unrecognized status: " + String(status || "") };
|
||||
}
|
||||
|
||||
export function extractUsageOnComplete(task, result, body) {
|
||||
// Legacy ratio pricing calls this hook without a poll body.
|
||||
const data = plainObject(body);
|
||||
if (!data || typeof data.status !== "string") return null;
|
||||
const facts = {};
|
||||
if (filteredOutput(data) || chargedFailure(data)) facts.moderated = true;
|
||||
// The state this poll returned arrives with its result; earlier polls left
|
||||
// theirs on the task.
|
||||
const state = plainObject(result && result.plugin_state) || plainObject(task && task.state) || {};
|
||||
if (VIDEO_MODELS["grok-imagine-video"].resolutions.includes(state.output_resolution)) facts.resolution = state.output_resolution;
|
||||
const duration = Number(data.status === "done" && plainObject(data.video) ? data.video.duration : NaN);
|
||||
if (!(duration > 0)) return facts;
|
||||
const action = task && task.action;
|
||||
if (action === "video_edit") {
|
||||
facts.seconds = Math.min(duration, MAX_EDIT_SECONDS);
|
||||
facts.input_video_seconds = facts.seconds;
|
||||
} else if (action === "video_extension") {
|
||||
// The extension length was reserved as seconds; the rest of the output is
|
||||
// the input video.
|
||||
const extension = Number(state.extension_seconds);
|
||||
if (extension > 0) facts.input_video_seconds = Math.min(Math.max(duration - extension, 0), MAX_EXTENSION_INPUT_SECONDS);
|
||||
} else {
|
||||
facts.seconds = Math.min(duration, MAX_DURATION);
|
||||
}
|
||||
return facts;
|
||||
}
|
||||
|
||||
// xAI bills the output of an edit or extension by its resolution but reports
|
||||
// neither it nor the source's, so the output's MP4 header is read: the moov
|
||||
// box, reached by stepping over the top-level boxes with ranged reads, holds a
|
||||
// tkhd per track with the track's size.
|
||||
const PROBE_READ_BYTES = 64 * 1024;
|
||||
const PROBE_MAX_BYTES = 1024 * 1024; // the utils.fetch body limit
|
||||
const PROBE_MAX_READS = 3; // utils.fetch allows 4 per hook call
|
||||
|
||||
function outputResolution(url) {
|
||||
try {
|
||||
const track = videoTrack(url);
|
||||
if (!track) return "";
|
||||
return Math.min(track.width, track.height) > 480 ? "720p" : "480p";
|
||||
} catch (_error) {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
|
||||
// Reads bytes [start, start + length) of the file. A server that ignores
|
||||
// Range answers 200 with the whole file, bounded by the fetch body limit.
|
||||
function readBytes(url, start, length) {
|
||||
const response = utils.fetch({ url: url, headers: { Range: "bytes=" + start + "-" + (start + length - 1) }, responseType: "bytes" });
|
||||
if (response.status !== 206 && response.status !== 200) throw new Error("HTTP " + response.status);
|
||||
let first = 0;
|
||||
if (response.status === 206) {
|
||||
const range = /^bytes (\d+)-/.exec(response.headers["Content-Range"] || "");
|
||||
first = range ? Number(range[1]) : start;
|
||||
}
|
||||
return { start: first, bytes: new Uint8Array(response.body) };
|
||||
}
|
||||
|
||||
function be32(bytes, offset) {
|
||||
return bytes[offset] * 16777216 + bytes[offset + 1] * 65536 + bytes[offset + 2] * 256 + bytes[offset + 3];
|
||||
}
|
||||
|
||||
// An ISO BMFF box header: a 32-bit size and a four-character type, with a
|
||||
// 64-bit size following when the size is 1; a size of 0 reaches end.
|
||||
function boxHeader(bytes, offset, end) {
|
||||
if (offset + 8 > end) return null;
|
||||
let size = be32(bytes, offset);
|
||||
let header = 8;
|
||||
if (size === 1) {
|
||||
if (offset + 16 > end) return null;
|
||||
size = be32(bytes, offset + 8) * 4294967296 + be32(bytes, offset + 12);
|
||||
header = 16;
|
||||
}
|
||||
const open = size === 0;
|
||||
if (open) size = end - offset;
|
||||
if (size < header) return null;
|
||||
return { type: String.fromCharCode(bytes[offset + 4], bytes[offset + 5], bytes[offset + 6], bytes[offset + 7]), size: size, header: header, open: open };
|
||||
}
|
||||
|
||||
function videoTrack(url) {
|
||||
let reads = 0;
|
||||
let chunk = { start: 0, bytes: new Uint8Array(0) };
|
||||
const read = function (start, length) {
|
||||
if (reads >= PROBE_MAX_READS) throw new Error("too many reads");
|
||||
reads++;
|
||||
chunk = readBytes(url, start, length);
|
||||
};
|
||||
let offset = 0;
|
||||
for (let i = 0; i < 32; i++) {
|
||||
if (offset < chunk.start || offset + 16 > chunk.start + chunk.bytes.length) read(offset, PROBE_READ_BYTES);
|
||||
let local = offset - chunk.start;
|
||||
let box = local >= 0 ? boxHeader(chunk.bytes, local, chunk.bytes.length) : null;
|
||||
if (!box) return null;
|
||||
if (box.type !== "moov") {
|
||||
offset += box.size;
|
||||
continue;
|
||||
}
|
||||
if (box.open || local + box.size > chunk.bytes.length) {
|
||||
read(offset, box.open ? PROBE_MAX_BYTES : Math.min(box.size, PROBE_MAX_BYTES));
|
||||
local = offset - chunk.start;
|
||||
box = local >= 0 ? boxHeader(chunk.bytes, local, chunk.bytes.length) : null;
|
||||
if (!box) return null;
|
||||
}
|
||||
return moovVideoTrack(chunk.bytes, local + box.header, Math.min(local + box.size, chunk.bytes.length));
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
// The largest track with a size is the video; audio tracks have none.
|
||||
function moovVideoTrack(bytes, start, end) {
|
||||
let best = null;
|
||||
let offset = start;
|
||||
while (true) {
|
||||
const box = boxHeader(bytes, offset, end);
|
||||
if (!box || offset + box.size > end) return best;
|
||||
if (box.type === "trak") {
|
||||
const track = trackSize(bytes, offset + box.header, offset + box.size);
|
||||
if (track && (!best || track.width * track.height > best.width * best.height)) best = track;
|
||||
}
|
||||
offset += box.size;
|
||||
}
|
||||
}
|
||||
|
||||
// tkhd stores the track's width and height as 16.16 fixed-point numbers after
|
||||
// fields whose size depends on its version.
|
||||
function trackSize(bytes, start, end) {
|
||||
let offset = start;
|
||||
while (true) {
|
||||
const box = boxHeader(bytes, offset, end);
|
||||
if (!box || offset + box.size > end) return null;
|
||||
if (box.type === "tkhd") {
|
||||
const fields = offset + box.header;
|
||||
const width = fields + 4 + (bytes[fields] === 1 ? 84 : 72);
|
||||
if (width + 8 > offset + box.size) return null;
|
||||
const track = { width: bytes[width] * 256 + bytes[width + 1], height: bytes[width + 4] * 256 + bytes[width + 5] };
|
||||
return track.width > 0 && track.height > 0 ? track : null;
|
||||
}
|
||||
offset += box.size;
|
||||
}
|
||||
}
|
||||
|
||||
export function listArtifacts(task) {
|
||||
return task.status === "SUCCESS" && videoURL(task.data) ? [{ key: "video", type: "video", mimeType: "video/mp4" }] : [];
|
||||
}
|
||||
|
||||
export function buildContentRequest(ctx) {
|
||||
if (ctx.artifactKey !== "video") throw new Error("artifact_not_found");
|
||||
const url = videoURL(ctx.data);
|
||||
if (!url) throw new Error("artifact_not_found");
|
||||
return { url: url, method: ctx.clientRequest.method, credentialless: true };
|
||||
}
|
||||
|
||||
export const protocols = {
|
||||
openai_responses: {
|
||||
decodeRequest: function (ctx) {
|
||||
if (!ctx.body || ctx.body.kind !== "json") throw new Error("JSON body required");
|
||||
const req = ctx.body.value;
|
||||
if (!req || typeof req !== "object" || Array.isArray(req)) throw new Error("request body must be an object");
|
||||
const model = trimmed(req.model);
|
||||
if (!model) throw new Error("model is required");
|
||||
if (req.input !== undefined && typeof req.input !== "string" && !Array.isArray(req.input)) throw new Error("input must be a string or array");
|
||||
if (req.images !== undefined && !Array.isArray(req.images)) throw new Error("images must be an array");
|
||||
if (req.metadata !== undefined && (!req.metadata || typeof req.metadata !== "object" || Array.isArray(req.metadata)))
|
||||
throw new Error("metadata must be an object");
|
||||
const input = responsesInput(req);
|
||||
const images = [];
|
||||
for (const image of [req.image, req.input_reference].concat(req.images || [], input.images)) {
|
||||
if (typeof image === "string" && image.trim() && !images.includes(image.trim())) images.push(image.trim());
|
||||
}
|
||||
// metadata carries xAI-only fields such as keyframes or generate_audio.
|
||||
const source = { prompt: input.prompt || trimmed(req.prompt), images: images };
|
||||
for (const key of ["metadata", "seconds", "duration", "size", "aspect_ratio", "resolution"]) {
|
||||
if (req[key] !== undefined) source[key] = req[key];
|
||||
}
|
||||
if (!source.prompt && !images.length) throw new Error("input is required");
|
||||
const request = videoRequest("generation", source, { upstreamModel: ctx.upstreamModel, model: model });
|
||||
return { kind: "submit", model: model, action: request.action, requestBody: Object.assign(request.body, { model: model }) };
|
||||
},
|
||||
renderEvents: function (ctx, task, previousState) {
|
||||
const status = String(task.status || "UNKNOWN").toUpperCase();
|
||||
const value = Number(String(task.progress || "").replace("%", ""));
|
||||
const progress = Number.isFinite(value) && value >= 0 && value <= 100 ? value : null;
|
||||
const state = { status: status, progress: progress };
|
||||
if (status === "SUCCESS") {
|
||||
const text = responsesVideoText(ctx, task);
|
||||
const events = previousState && previousState.status === status ? [] : [{ type: "output", data: text }];
|
||||
return { events: events, state: state, done: true };
|
||||
}
|
||||
if (status === "FAILURE")
|
||||
return { events: [{ type: "error", code: "task_failed", message: task.fail_reason || "task failed" }], state: state, done: true };
|
||||
if (previousState && previousState.status === status && previousState.progress === progress) return { events: [], state: state, done: false };
|
||||
const event = { type: "progress", message: status.toLowerCase() };
|
||||
if (progress !== null) event.progress = progress;
|
||||
return { events: [event], state: state, done: false };
|
||||
},
|
||||
renderFinal: function (ctx, task) {
|
||||
return {
|
||||
output: [
|
||||
{
|
||||
type: "message",
|
||||
status: "completed",
|
||||
role: "assistant",
|
||||
content: [{ type: "output_text", text: responsesVideoText(ctx, task), annotations: [], logprobs: [] }],
|
||||
},
|
||||
],
|
||||
metadata: { vendor: "xai" },
|
||||
};
|
||||
},
|
||||
},
|
||||
openai_video: {
|
||||
decodeRequest: function (ctx) {
|
||||
if (!ctx.body || (ctx.body.kind !== "json" && ctx.body.kind !== "multipart")) throw new Error("JSON or multipart body required");
|
||||
let req;
|
||||
if (ctx.body.kind === "json") {
|
||||
req = plainObject(ctx.body.value);
|
||||
if (!req) throw new Error("JSON object required");
|
||||
} else {
|
||||
req = {};
|
||||
const fields = ctx.body.fields || {};
|
||||
for (const name of Object.keys(fields)) {
|
||||
if (fields[name].length > 1) throw new Error(name + " must be provided once");
|
||||
req[name] = fields[name][0];
|
||||
}
|
||||
if (req.generate_audio !== undefined) {
|
||||
if (req.generate_audio !== "true" && req.generate_audio !== "false") throw new Error("generate_audio must be true or false");
|
||||
req.generate_audio = req.generate_audio === "true";
|
||||
}
|
||||
const files = ctx.body.files || [];
|
||||
for (const file of files) {
|
||||
if (file.field !== "input_reference") throw new Error("unexpected file field: " + file.field);
|
||||
}
|
||||
if (files.length > 1) throw new Error("input_reference must be provided once");
|
||||
if (files.length) req.input_reference = { url: { __fileRef: files[0].ref, encoding: "dataUrl" } };
|
||||
}
|
||||
const request = videoRequest("generation", req, { upstreamModel: ctx.upstreamModel, model: ctx.model });
|
||||
return { kind: "submit", model: ctx.model, action: request.action, requestBody: Object.assign(request.body, { model: ctx.model }) };
|
||||
},
|
||||
// The host owns id, status, progress, and timestamps; a moderated video
|
||||
// reports completed (it is charged) and explains the missing content here.
|
||||
render: function (ctx, task) {
|
||||
const data = plainObject(task.data) || {};
|
||||
const output = { object: "video" };
|
||||
const seconds = Number(plainObject(data.video) ? data.video.duration : NaN);
|
||||
if (seconds > 0) output.seconds = String(seconds);
|
||||
const notice = task.status === "SUCCESS" ? moderationNotice(data) : "";
|
||||
if (notice) output.error = { code: "moderation_blocked", message: notice };
|
||||
if (task.status === "FAILURE")
|
||||
output.error = {
|
||||
code: trimmed(plainObject(data.error) && data.error.code) || "video_generation_failed",
|
||||
message: task.fail_reason || failureReason(data),
|
||||
};
|
||||
return output;
|
||||
},
|
||||
},
|
||||
};
|
||||
@@ -201,3 +201,80 @@ func decodePluginValue(t *testing.T, value any) map[string]any {
|
||||
require.NoError(t, common.Unmarshal(encoded, &decoded))
|
||||
return decoded
|
||||
}
|
||||
|
||||
// A New API channel reaches Vertex AI through another gateway's /vertex-ai
|
||||
// route, which runs the request in the gateway's own project. Operation names
|
||||
// carry the caller's path and the gateway task id, never the gateway's project,
|
||||
// and fetchPredictOperation on such a name reads that gateway task.
|
||||
func TestVertexAIThroughNewAPIGateway(t *testing.T) {
|
||||
source, err := builtinplugins.Source("vertex-ai")
|
||||
require.NoError(t, err)
|
||||
plugin, err := jsplugin.NewRegistry().RegisterFactory(source, jsplugin.Options{Key: "vertex-ai"})
|
||||
require.NoError(t, err)
|
||||
call := func(export string, members []string, args ...any) map[string]any {
|
||||
t.Helper()
|
||||
value, err := plugin.Engine.CallPath(t.Context(), export, members, args...)
|
||||
require.NoError(t, err)
|
||||
return decodePluginValue(t, value)
|
||||
}
|
||||
const veo = "veo-3.0-generate-001"
|
||||
const route = "https://gateway.example/vertex-ai/v1/projects/-/locations/global/publishers/google/models/" + veo
|
||||
gateway := map[string]any{"kind": "new_api"}
|
||||
gatewayAuth := map[string]any{"authHeader": "Bearer sk-gateway"}
|
||||
|
||||
downstream := map[string]any{
|
||||
"requestBody": map[string]any{"model": veo, "prompt": "p", "duration": 6, "images": []any{"data:image/png;base64,aGVsbG8="},
|
||||
"metadata": map[string]any{"resolution": "1080p", "generateAudio": false}},
|
||||
"model": veo, "upstreamModel": veo, "baseUrl": "https://gateway.example", "apiKey": "sk-gateway",
|
||||
"authHeader": "Bearer sk-gateway", "auth": gatewayAuth, "upstream": gateway,
|
||||
}
|
||||
sent := call("buildSubmitRequest", nil, downstream)
|
||||
assert.Equal(t, route+":predictLongRunning", sent["url"])
|
||||
assert.Equal(t, map[string]any{"Accept": "application/json", "Content-Type": "application/json", "Authorization": "Bearer sk-gateway"}, sent["headers"])
|
||||
|
||||
submitParams := map[string]any{"project": "-", "location": "global", "model": veo + ":predictLongRunning"}
|
||||
intent := call("native", []string{"modelMethod"}, map[string]any{"params": submitParams, "body": map[string]any{"kind": "json", "value": sent["body"]}})
|
||||
assert.Equal(t, "submit", intent["kind"])
|
||||
assert.Equal(t, sent["action"], intent["action"])
|
||||
upstream := map[string]any{"requestBody": intent["requestBody"], "model": veo, "upstreamModel": veo, "baseUrl": "https://aiplatform.example",
|
||||
"authHeader": "Bearer ya29.vendor", "auth": map[string]any{"authHeader": "Bearer ya29.vendor", "projectId": "gateway-project"},
|
||||
"upstream": map[string]any{"kind": "vendor"}}
|
||||
forwarded := call("buildSubmitRequest", nil, upstream)
|
||||
assert.Equal(t, "https://aiplatform.example/v1/projects/gateway-project/locations/global/publishers/google/models/"+veo+":predictLongRunning", forwarded["url"])
|
||||
assert.Equal(t, "gateway-project", forwarded["headers"].(map[string]any)["x-goog-user-project"])
|
||||
assert.Equal(t, sent["body"], forwarded["body"])
|
||||
assert.Equal(t, call("extractUsage", nil, downstream), call("extractUsage", nil, upstream))
|
||||
|
||||
vendorOperation := "projects/gateway-project/locations/us-central1/publishers/google/models/" + veo + "/operations/op-private"
|
||||
created := call("native", []string{"operation"}, map[string]any{"params": submitParams},
|
||||
map[string]any{"task_id": "task_pub", "status": "SUBMITTED", "data": map[string]any{"name": vendorOperation}})
|
||||
assert.Equal(t, map[string]any{"name": "projects/-/locations/global/publishers/google/models/" + veo + "/operations/task_pub"}, created)
|
||||
upstreamTaskID := call("parseSubmitResponse", nil, map[string]any{}, map[string]any{"body": created})["taskId"]
|
||||
query := call("buildQueryRequest", nil, map[string]any{"taskId": upstreamTaskID, "baseUrl": "https://gateway.example",
|
||||
"authHeader": "Bearer sk-gateway", "auth": gatewayAuth, "upstream": gateway})
|
||||
assert.Equal(t, route+":fetchPredictOperation", query["url"])
|
||||
fetchParams := map[string]any{"project": "-", "location": "global", "model": veo + ":fetchPredictOperation"}
|
||||
assert.Equal(t, map[string]any{"kind": "query", "taskIds": []any{"task_pub"}},
|
||||
call("native", []string{"modelMethod"}, map[string]any{"params": fetchParams, "body": map[string]any{"kind": "json", "value": query["body"]}}))
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
task map[string]any
|
||||
wantStatus string
|
||||
wantReason string
|
||||
}{
|
||||
{"running", map[string]any{"status": "IN_PROGRESS", "data": map[string]any{"name": vendorOperation}}, "IN_PROGRESS", ""},
|
||||
{"succeeded", map[string]any{"status": "SUCCESS", "data": map[string]any{"name": vendorOperation, "done": true,
|
||||
"response": map[string]any{"videos": []any{map[string]any{"mimeType": "video/mp4"}}}}}, "SUCCESS", ""},
|
||||
{"failed by the gateway poller", map[string]any{"status": "FAILURE", "fail_reason": "task timed out", "data": map[string]any{"name": vendorOperation}}, "FAILURE", "task timed out"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.task["task_id"] = "task_pub"
|
||||
rendered := call("native", []string{"operation"}, map[string]any{"params": fetchParams}, []any{tc.task})
|
||||
assert.Equal(t, "projects/-/locations/global/publishers/google/models/"+veo+"/operations/task_pub", rendered["name"])
|
||||
result := call("parseTaskResult", nil, map[string]any{}, rendered)
|
||||
assert.Equal(t, tc.wantStatus, result["status"])
|
||||
assert.Equal(t, tc.wantReason, common.Interface2String(result["reason"]))
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,15 @@
|
||||
package plugins_test
|
||||
|
||||
import "testing"
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/pkg/jsplugin"
|
||||
builtinplugins "github.com/QuantumNous/new-api/plugins"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestViduResponsesProtocol(t *testing.T) {
|
||||
testVideoResponsesProtocol(t, videoResponsesTestCase{
|
||||
@@ -29,3 +38,120 @@ func TestViduResponsesProtocol(t *testing.T) {
|
||||
wantVendorName: "vidu",
|
||||
})
|
||||
}
|
||||
|
||||
// metadata carries Vidu's other fields but cannot replace the model, duration
|
||||
// or resolution the request is billed for.
|
||||
func TestViduMetadataCannotChangeWhatIsBilled(t *testing.T) {
|
||||
source, err := builtinplugins.Source("vidu")
|
||||
require.NoError(t, err)
|
||||
plugin, err := jsplugin.NewRegistry().RegisterFactory(source, jsplugin.Options{Key: "vidu"})
|
||||
require.NoError(t, err)
|
||||
ctx := map[string]any{"baseUrl": "https://api.vidu.example", "apiKey": "vendor-key", "model": "vidu1.5", "upstreamModel": "vidu1.5",
|
||||
"requestBody": map[string]any{"model": "vidu1.5", "prompt": "p", "duration": 4, "resolution": "360p",
|
||||
"metadata": map[string]any{"model": "viduq2", "duration": 8, "resolution": "1080p", "seed": 7}}}
|
||||
descriptor, err := plugin.Engine.Call(t.Context(), "buildSubmitRequest", ctx)
|
||||
require.NoError(t, err)
|
||||
usage, err := plugin.Engine.Call(t.Context(), "extractUsage", ctx)
|
||||
require.NoError(t, err)
|
||||
body := decodePluginValue(t, descriptor)["body"].(map[string]any)
|
||||
assert.Equal(t, map[string]any{"duration": float64(4), "resolution": "360p"}, decodePluginValue(t, usage))
|
||||
assert.Equal(t, "vidu1.5", body["model"])
|
||||
assert.Equal(t, float64(4), body["duration"])
|
||||
assert.Equal(t, "360p", body["resolution"])
|
||||
assert.Equal(t, float64(7), body["seed"])
|
||||
}
|
||||
|
||||
// A New API channel reaches Vidu through another gateway's /vidu routes. The
|
||||
// gateway rebuilds the request this driver sent, so Vidu receives one body and
|
||||
// both gateways bill the same facts, and the gateway's task reads back as the
|
||||
// same state and credits.
|
||||
func TestViduThroughNewAPIGateway(t *testing.T) {
|
||||
source, err := builtinplugins.Source("vidu")
|
||||
require.NoError(t, err)
|
||||
plugin, err := jsplugin.NewRegistry().RegisterFactory(source, jsplugin.Options{Key: "vidu"})
|
||||
require.NoError(t, err)
|
||||
call := func(export string, members []string, args ...any) map[string]any {
|
||||
t.Helper()
|
||||
value, err := plugin.Engine.CallPath(t.Context(), export, members, args...)
|
||||
require.NoError(t, err)
|
||||
return decodePluginValue(t, value)
|
||||
}
|
||||
gateway := map[string]any{"kind": "new_api"}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
requestBody map[string]any
|
||||
route string
|
||||
}{
|
||||
{
|
||||
name: "text to video with vendor fields",
|
||||
requestBody: map[string]any{"model": "viduq2", "prompt": "p", "duration": 8, "size": "1280x720",
|
||||
"metadata": map[string]any{"bgm": true, "seed": 7, "aspect_ratio": "9:16"}},
|
||||
route: "/vidu/ent/v2/text2video",
|
||||
},
|
||||
{
|
||||
name: "two reference images stay a reference request",
|
||||
requestBody: map[string]any{"model": "viduq2", "prompt": "p", "images": []any{"https://cdn.example/a.png", "https://cdn.example/b.png"},
|
||||
"metadata": map[string]any{"action": "referenceGenerate"}},
|
||||
route: "/vidu/ent/v2/reference2video",
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
downstream := map[string]any{"requestBody": tc.requestBody, "model": "viduq2", "upstreamModel": "viduq2",
|
||||
"baseUrl": "https://gateway.example", "apiKey": "sk-gateway", "upstream": gateway}
|
||||
sent := call("buildSubmitRequest", nil, downstream)
|
||||
assert.Equal(t, "https://gateway.example"+tc.route, sent["url"])
|
||||
assert.Equal(t, map[string]any{"Accept": "application/json", "Content-Type": "application/json", "Authorization": "Bearer sk-gateway"}, sent["headers"])
|
||||
|
||||
intent := call("native", []string{"createTask"}, map[string]any{"path": tc.route, "body": map[string]any{"kind": "json", "value": sent["body"]}})
|
||||
assert.Equal(t, sent["action"], intent["action"])
|
||||
upstream := map[string]any{"requestBody": intent["requestBody"], "model": "viduq2", "upstreamModel": "viduq2",
|
||||
"baseUrl": "https://api.vidu.example", "apiKey": "vendor-key", "upstream": map[string]any{"kind": "vendor"}}
|
||||
forwarded := call("buildSubmitRequest", nil, upstream)
|
||||
assert.Equal(t, "https://api.vidu.example"+strings.TrimPrefix(tc.route, "/vidu"), forwarded["url"])
|
||||
assert.Equal(t, sent["body"], forwarded["body"])
|
||||
assert.Equal(t, call("extractUsage", nil, downstream), call("extractUsage", nil, upstream))
|
||||
})
|
||||
}
|
||||
|
||||
created := call("native", []string{"taskCreated"}, map[string]any{}, map[string]any{"task_id": "task_pub", "status": "SUBMITTED",
|
||||
"data": map[string]any{"task_id": "task_pub", "state": "created", "credits": 10}})
|
||||
assert.Equal(t, "task_pub", call("parseSubmitResponse", nil, map[string]any{}, map[string]any{"body": created})["taskId"])
|
||||
|
||||
video := "https://cdn.vidu.example/v.mp4"
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
task map[string]any
|
||||
wantStatus string
|
||||
wantReason string
|
||||
}{
|
||||
{
|
||||
name: "before the first poll", wantStatus: "SUBMITTED",
|
||||
task: map[string]any{"status": "SUBMITTED", "data": map[string]any{"task_id": "task_pub", "state": "created", "credits": 10}},
|
||||
},
|
||||
{
|
||||
name: "succeeded", wantStatus: "SUCCESS",
|
||||
task: map[string]any{"status": "SUCCESS", "data": map[string]any{"id": "task_pub", "state": "success", "credits": 12,
|
||||
"creations": []any{map[string]any{"id": "c1", "url": video}}}},
|
||||
},
|
||||
{
|
||||
name: "failed by the gateway poller", wantStatus: "FAILURE", wantReason: "task timed out",
|
||||
task: map[string]any{"status": "FAILURE", "fail_reason": "task timed out", "data": map[string]any{"id": "task_pub", "state": "processing"}},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
tc.task["task_id"] = "task_pub"
|
||||
rendered := call("native", []string{"taskStatus"}, map[string]any{}, tc.task)
|
||||
result := call("parseTaskResult", nil, map[string]any{}, rendered)
|
||||
assert.Equal(t, tc.wantStatus, result["status"])
|
||||
assert.Equal(t, tc.wantReason, common.Interface2String(result["reason"]))
|
||||
if tc.wantStatus != "SUCCESS" {
|
||||
return
|
||||
}
|
||||
assert.Equal(t, map[string]any{"credits": float64(12)}, call("extractUsageOnComplete", nil, nil, nil, rendered))
|
||||
content := call("buildContentRequest", nil, map[string]any{"artifactKey": "video", "data": rendered, "upstream": gateway,
|
||||
"clientRequest": map[string]any{"method": "GET", "headers": map[string]any{}}})
|
||||
assert.Equal(t, map[string]any{"url": video, "method": "GET", "credentialless": true}, content)
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,484 @@
|
||||
package plugins_test
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/pkg/billingexpr"
|
||||
"github.com/QuantumNous/new-api/pkg/jsplugin"
|
||||
builtinplugins "github.com/QuantumNous/new-api/plugins"
|
||||
"github.com/QuantumNous/new-api/relay/channel"
|
||||
taskplugin "github.com/QuantumNous/new-api/relay/channel/task/jsplugin"
|
||||
relaycommon "github.com/QuantumNous/new-api/relay/common"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
"github.com/QuantumNous/new-api/setting/billing_setting"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func newXAIPlugin(t *testing.T) *jsplugin.LoadedPlugin {
|
||||
t.Helper()
|
||||
source, err := builtinplugins.Source("xai")
|
||||
require.NoError(t, err)
|
||||
plugin, err := jsplugin.NewRegistry().RegisterFactory(source, jsplugin.Options{Key: "xai"})
|
||||
require.NoError(t, err)
|
||||
return plugin
|
||||
}
|
||||
|
||||
func callXAI(t *testing.T, plugin *jsplugin.LoadedPlugin, path []string, args ...any) (map[string]any, error) {
|
||||
t.Helper()
|
||||
var value any
|
||||
var err error
|
||||
if len(path) == 1 {
|
||||
value, err = plugin.Engine.Call(t.Context(), path[0], args...)
|
||||
} else {
|
||||
value, err = plugin.Engine.CallPath(t.Context(), path[0], path[1:], args...)
|
||||
}
|
||||
if err != nil || value == nil {
|
||||
return nil, err
|
||||
}
|
||||
return alibabaObject(t, value), nil
|
||||
}
|
||||
|
||||
func TestXAIResponsesProtocol(t *testing.T) {
|
||||
testVideoResponsesProtocol(t, videoResponsesTestCase{
|
||||
pluginKey: "xai",
|
||||
model: "grok-imagine-video-1.5",
|
||||
requestBody: map[string]any{
|
||||
"model": "grok-imagine-video-1.5",
|
||||
"input": []any{map[string]any{"role": "user", "content": []any{
|
||||
map[string]any{"type": "input_text", "text": "a fox runs through snow"},
|
||||
map[string]any{"type": "input_image", "image_url": "https://cdn.example/fox.png"},
|
||||
}}},
|
||||
"seconds": 10,
|
||||
"size": "1280x720",
|
||||
},
|
||||
wantAction: "image_to_video",
|
||||
wantRequest: map[string]any{
|
||||
"model": "grok-imagine-video-1.5",
|
||||
"prompt": "a fox runs through snow",
|
||||
"image": map[string]any{"url": "https://cdn.example/fox.png"},
|
||||
"duration": float64(10),
|
||||
"resolution": "720p",
|
||||
"aspect_ratio": "16:9",
|
||||
},
|
||||
wantUsageKeys: []string{"input_images", "input_video_seconds", "moderated", "resolution", "seconds"},
|
||||
wantSubmitUsageKeys: []string{"input_images", "moderated", "resolution", "seconds"},
|
||||
wantVendorName: "xai",
|
||||
})
|
||||
}
|
||||
|
||||
// xAI still bills a generation its moderation blocks, and the host refunds
|
||||
// every FAILURE task in full, so a blocked video must settle as a charged
|
||||
// SUCCESS. The flow runs through the production adaptor and settlement
|
||||
// evaluator with a price an administrator can save.
|
||||
func TestXAIModeratedVideoKeepsItsCharge(t *testing.T) {
|
||||
plugin := newXAIPlugin(t)
|
||||
const videoModel = "grok-imagine-video-1.5"
|
||||
const expression = `u("moderated") == true ? tier("blocked", 0.05 + u("seconds") * 0) : ` +
|
||||
`u("resolution") == "1080p" ? tier("1080p", u("seconds") * 0.25 + u("input_images") * 0.01) : ` +
|
||||
`u("resolution") == "720p" ? tier("720p", u("seconds") * 0.14 + u("input_images") * 0.01) : ` +
|
||||
`tier("480p", u("seconds") * 0.08 + u("input_images") * 0.01)`
|
||||
schema, _ := plugin.Meta.UsageForModel(videoModel)
|
||||
require.NoError(t, billing_setting.SmokeTestTaskExpr(expression, schema))
|
||||
|
||||
info := &relaycommon.RelayInfo{
|
||||
ChannelMeta: &relaycommon.ChannelMeta{ChannelBaseUrl: "https://api.x.ai", ApiKey: "xai-key", UpstreamModelName: videoModel},
|
||||
OriginModelName: videoModel,
|
||||
TaskRelayInfo: &relaycommon.TaskRelayInfo{PublicTaskID: "task_public", Action: "image_to_video"},
|
||||
}
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(info)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/v1/videos", nil)
|
||||
c.Set("task_request", map[string]any{"model": videoModel, "prompt": "a fox", "input_reference": "https://cdn.example/fox.png", "seconds": "8", "size": "1280x720"})
|
||||
require.Nil(t, adaptor.ValidateRequestAndSetAction(c, info))
|
||||
facts, err := adaptor.ExtractUsageFactsValidated(c, info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"seconds": float64(8), "resolution": "720p", "input_images": float64(1), "moderated": false}, facts)
|
||||
_, trace, err := billingexpr.RunExprWithRequest(expression, billingexpr.TokenParams{}, billingexpr.RequestInput{Usage: facts})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "720p", trace.MatchedTier)
|
||||
snapshot := &billingexpr.BillingSnapshot{
|
||||
BillingMode: billing_setting.BillingModeTieredExpr, ModelName: videoModel, ExprString: expression, ExprHash: billingexpr.ExprHashString(expression),
|
||||
GroupRatio: 1, QuotaPerUnit: 500000, ExprVersion: billingexpr.ExprVersion(expression), TaskUsageBilling: true, UsageFacts: facts,
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
httpStatus int
|
||||
body string
|
||||
wantStatus string
|
||||
wantProgress string
|
||||
wantReason string
|
||||
wantFacts map[string]any
|
||||
wantTier string
|
||||
wantQuota int
|
||||
wantVideo bool
|
||||
}{
|
||||
{name: "still rendering", httpStatus: http.StatusAccepted, body: ``, wantStatus: "IN_PROGRESS"},
|
||||
{name: "progress", httpStatus: http.StatusOK, body: `{"status":"pending","progress":42}`, wantStatus: "IN_PROGRESS", wantProgress: "42%"},
|
||||
{
|
||||
name: "delivered video settles on its length", httpStatus: http.StatusOK,
|
||||
body: `{"status":"done","progress":100,"model":"grok-imagine-video-1.5","video":{"url":"https://vidgen.x.ai/v.mp4","duration":6,"respect_moderation":true},"usage":{"cost_in_usd_ticks":8500000000}}`,
|
||||
wantStatus: "SUCCESS", wantFacts: map[string]any{"seconds": float64(6)}, wantTier: "720p", wantQuota: 425000, wantVideo: true,
|
||||
},
|
||||
{
|
||||
name: "moderated output is charged", httpStatus: http.StatusOK,
|
||||
body: `{"status":"done","progress":100,"video":{"url":"","duration":8,"respect_moderation":false},"usage":{"cost_in_usd_ticks":11200000000}}`,
|
||||
wantStatus: "SUCCESS", wantFacts: map[string]any{"seconds": float64(8), "moderated": true}, wantTier: "blocked", wantQuota: 25000,
|
||||
},
|
||||
{
|
||||
name: "failure xAI billed is charged", httpStatus: http.StatusOK,
|
||||
body: `{"status":"failed","error":{"code":"invalid_argument","message":"Content blocked by moderation"},"usage":{"cost_in_usd_ticks":500000000}}`,
|
||||
wantStatus: "SUCCESS", wantFacts: map[string]any{"moderated": true}, wantTier: "blocked", wantQuota: 25000,
|
||||
},
|
||||
{
|
||||
name: "unbilled failure is refunded", httpStatus: http.StatusOK,
|
||||
body: `{"status":"failed","error":{"code":"internal_error","message":"engine crashed"},"usage":{"cost_in_usd_ticks":0}}`,
|
||||
wantStatus: "FAILURE", wantReason: "engine crashed",
|
||||
},
|
||||
{name: "expired result", httpStatus: http.StatusOK, body: `{"status":"expired"}`, wantStatus: "FAILURE", wantReason: "the video expired before it was retrieved"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
task := &model.Task{
|
||||
TaskID: "task_public", Action: "image_to_video", Data: []byte(tc.body),
|
||||
Properties: model.Properties{OriginModelName: videoModel, UpstreamModelName: videoModel},
|
||||
}
|
||||
result, err := adaptor.ParseTaskResult(task, &http.Response{StatusCode: tc.httpStatus, Header: make(http.Header)}, []byte(tc.body))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantStatus, result.Status)
|
||||
assert.Equal(t, tc.wantProgress, result.Progress)
|
||||
assert.Equal(t, tc.wantReason, result.Reason)
|
||||
if tc.wantStatus == "IN_PROGRESS" {
|
||||
return
|
||||
}
|
||||
// A downstream New API gateway polls this task's native status
|
||||
// route, which omits usage, and must settle on the same facts.
|
||||
var data any
|
||||
require.NoError(t, common.UnmarshalJsonStr(tc.body, &data))
|
||||
native, err := plugin.Engine.CallPath(t.Context(), "native", []string{"videoStatus"}, map[string]any{},
|
||||
map[string]any{"task_id": "task_public", "status": result.Status, "fail_reason": result.Reason, "data": data})
|
||||
require.NoError(t, err)
|
||||
nativeBody, err := common.Marshal(native)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(nativeBody), "cost_in_usd_ticks")
|
||||
downstream, err := adaptor.ParseTaskResult(task, &http.Response{StatusCode: http.StatusOK, Header: make(http.Header)}, nativeBody)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, result.Status, downstream.Status)
|
||||
assert.Equal(t, result.UsageFacts, downstream.UsageFacts)
|
||||
if tc.wantStatus != "SUCCESS" {
|
||||
return
|
||||
}
|
||||
assert.Equal(t, tc.wantFacts, result.UsageFacts)
|
||||
settled, _, err := service.EvaluateTaskCompletionUsage(snapshot, result.UsageFacts)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantTier, settled.MatchedTier)
|
||||
assert.Equal(t, tc.wantQuota, settled.ActualQuotaAfterGroup)
|
||||
|
||||
task.Status = model.TaskStatusSuccess
|
||||
artifacts, err := adaptor.ListArtifacts(task)
|
||||
require.NoError(t, err)
|
||||
if !tc.wantVideo {
|
||||
assert.Empty(t, artifacts, "a blocked result has no video to proxy")
|
||||
return
|
||||
}
|
||||
assert.Equal(t, []channel.TaskArtifact{{Key: "video", Type: "video", MimeType: "video/mp4"}}, artifacts)
|
||||
content, err := adaptor.BuildContentRequest(t.Context(), task, "video", channel.TaskArtifactClientRequest{Method: http.MethodGet})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "https://vidgen.x.ai/v.mp4", content.URL)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestXAIVideoRequests(t *testing.T) {
|
||||
plugin := newXAIPlugin(t)
|
||||
referenceImages := []any{map[string]any{"url": "https://cdn.example/a.png"}, map[string]any{"url": "https://cdn.example/b.png"}}
|
||||
inputVideo := map[string]any{"url": "https://cdn.example/in.mp4"}
|
||||
jsonBody := func(value map[string]any) map[string]any { return map[string]any{"kind": "json", "value": value} }
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
decoder []string
|
||||
ctx map[string]any
|
||||
wantModel string
|
||||
wantPath string
|
||||
wantBody map[string]any
|
||||
wantFacts map[string]any
|
||||
}{
|
||||
{
|
||||
name: "reference images keep only documented fields",
|
||||
decoder: []string{"native", "createGeneration"},
|
||||
ctx: map[string]any{"body": jsonBody(map[string]any{
|
||||
"model": "grok-imagine-video-1.5", "prompt": "<IMAGE_0> meets <IMAGE_1>", "reference_images": referenceImages, "resolution": "720p",
|
||||
"user": "end-user", "n": 4, "storage_options": map[string]any{"filename": "out.mp4"},
|
||||
})},
|
||||
wantModel: "grok-imagine-video-1.5",
|
||||
wantPath: "/v1/videos/generations",
|
||||
wantBody: map[string]any{
|
||||
"model": "grok-imagine-video-1.5", "prompt": "<IMAGE_0> meets <IMAGE_1>", "reference_images": referenceImages,
|
||||
"duration": float64(8), "resolution": "720p", "user": "end-user",
|
||||
},
|
||||
wantFacts: map[string]any{"seconds": float64(8), "resolution": "720p", "input_images": float64(2), "moderated": false},
|
||||
},
|
||||
{
|
||||
name: "edit reserves the longest input and drops output settings",
|
||||
decoder: []string{"native", "createEdit"},
|
||||
ctx: map[string]any{"body": jsonBody(map[string]any{
|
||||
"model": "grok-imagine-video", "prompt": "make it snow", "video": inputVideo, "duration": 5, "resolution": "720p",
|
||||
})},
|
||||
wantModel: "grok-imagine-video",
|
||||
wantPath: "/v1/videos/edits",
|
||||
wantBody: map[string]any{"model": "grok-imagine-video", "prompt": "make it snow", "video": inputVideo},
|
||||
wantFacts: map[string]any{"seconds": float64(9), "resolution": "720p", "input_images": float64(0), "input_video_seconds": float64(9), "moderated": false},
|
||||
},
|
||||
{
|
||||
name: "extension bills the added seconds",
|
||||
decoder: []string{"native", "createExtension"},
|
||||
ctx: map[string]any{"body": jsonBody(map[string]any{
|
||||
"model": "grok-imagine-video", "prompt": "keep walking", "video": inputVideo, "duration": 4,
|
||||
})},
|
||||
wantModel: "grok-imagine-video",
|
||||
wantPath: "/v1/videos/extensions",
|
||||
wantBody: map[string]any{"model": "grok-imagine-video", "prompt": "keep walking", "video": inputVideo, "duration": float64(4)},
|
||||
wantFacts: map[string]any{"seconds": float64(4), "resolution": "720p", "input_images": float64(0), "input_video_seconds": float64(15), "moderated": false},
|
||||
},
|
||||
{
|
||||
name: "OpenAI multipart upload becomes the first frame",
|
||||
decoder: []string{"protocols", "openai_video", "decodeRequest"},
|
||||
ctx: map[string]any{"model": "grok-imagine-video", "body": map[string]any{
|
||||
"kind": "multipart",
|
||||
"fields": map[string]any{"prompt": []any{"a cat"}, "seconds": []any{"6"}, "size": []any{"720x1280"}},
|
||||
"files": []any{map[string]any{"ref": "request_file:input_reference", "field": "input_reference", "filename": "cat.png", "mimeType": "image/png", "size": 10}},
|
||||
}},
|
||||
wantModel: "grok-imagine-video",
|
||||
wantPath: "/v1/videos/generations",
|
||||
wantBody: map[string]any{
|
||||
"model": "grok-imagine-video", "prompt": "a cat", "duration": float64(6), "resolution": "720p", "aspect_ratio": "9:16",
|
||||
"image": map[string]any{"url": map[string]any{"__fileRef": "request_file:input_reference", "encoding": "dataUrl"}},
|
||||
},
|
||||
wantFacts: map[string]any{"seconds": float64(6), "resolution": "720p", "input_images": float64(1), "input_video_seconds": float64(0), "moderated": false},
|
||||
},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
intent, err := callXAI(t, plugin, tc.decoder, tc.ctx)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantModel, intent["model"])
|
||||
driver := map[string]any{
|
||||
"model": tc.wantModel, "upstreamModel": tc.wantModel, "action": intent["action"], "requestBody": intent["requestBody"],
|
||||
"baseUrl": "https://api.x.ai", "apiKey": "xai-key", "upstream": map[string]any{"kind": "vendor"},
|
||||
}
|
||||
descriptor, err := callXAI(t, plugin, []string{"buildSubmitRequest"}, driver)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "https://api.x.ai"+tc.wantPath, descriptor["url"])
|
||||
assert.Equal(t, tc.wantBody, descriptor["body"])
|
||||
facts, err := callXAI(t, plugin, []string{"extractUsage"}, driver)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantFacts, facts)
|
||||
})
|
||||
}
|
||||
|
||||
t.Run("legacy task shape reaches the driver undecoded", func(t *testing.T) {
|
||||
descriptor, err := callXAI(t, plugin, []string{"buildSubmitRequest"}, map[string]any{
|
||||
"model": "grok-imagine-video-1.5", "action": "text_to_video", "baseUrl": "https://api.x.ai", "apiKey": "xai-key",
|
||||
"requestBody": map[string]any{"prompt": "two cats", "images": []any{"https://cdn.example/a.png", "https://cdn.example/b.png"}, "seconds": "5", "size": "480p", "metadata": map[string]any{"generate_audio": false}},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "reference_to_video", descriptor["action"])
|
||||
assert.Equal(t, map[string]any{
|
||||
"model": "grok-imagine-video-1.5", "prompt": "two cats", "reference_images": referenceImages,
|
||||
"duration": float64(5), "resolution": "480p", "generate_audio": false,
|
||||
}, descriptor["body"])
|
||||
})
|
||||
|
||||
t.Run("completion settles edit and extension lengths", func(t *testing.T) {
|
||||
parsed, err := callXAI(t, plugin, []string{"parseSubmitResponse"}, map[string]any{"action": "video_extension", "requestBody": map[string]any{"duration": 4}}, map[string]any{"body": map[string]any{"request_id": "req-1"}})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"extension_seconds": float64(4)}, parsed["state"])
|
||||
done := map[string]any{"status": "done", "video": map[string]any{"url": "https://vidgen.x.ai/v.mp4", "duration": 12, "respect_moderation": true}}
|
||||
facts, err := callXAI(t, plugin, []string{"extractUsageOnComplete"}, map[string]any{"action": "video_extension", "state": parsed["state"]}, map[string]any{"status": "SUCCESS"}, done)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"input_video_seconds": float64(8)}, facts)
|
||||
facts, err = callXAI(t, plugin, []string{"extractUsageOnComplete"}, map[string]any{"action": "video_edit"}, map[string]any{"status": "SUCCESS"}, done)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"seconds": float64(9), "input_video_seconds": float64(9)}, facts)
|
||||
})
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
decoder string
|
||||
body map[string]any
|
||||
wantErr string
|
||||
}{
|
||||
{"edits need the video-input model", "createEdit", map[string]any{"model": "grok-imagine-video-1.5", "prompt": "x", "video": inputVideo}, "video editing and extension require grok-imagine-video"},
|
||||
{"model without 1080p", "createGeneration", map[string]any{"model": "grok-imagine-video", "prompt": "x", "resolution": "1080p"}, "grok-imagine-video resolution must be one of 480p, 720p"},
|
||||
{"reference-to-video is capped at 720p", "createGeneration", map[string]any{"model": "grok-imagine-video-1.5", "prompt": "x", "reference_images": referenceImages, "resolution": "1080p"}, "reference-to-video supports resolution 480p or 720p"},
|
||||
{"duration bound", "createGeneration", map[string]any{"model": "grok-imagine-video-1.5", "prompt": "x", "duration": 16}, "duration must be an integer between 1 and 15"},
|
||||
{"extension bound", "createExtension", map[string]any{"model": "grok-imagine-video", "prompt": "x", "video": inputVideo, "duration": 11}, "duration must be an integer between 2 and 10"},
|
||||
{"gateway files are unreachable", "createGeneration", map[string]any{"model": "grok-imagine-video-1.5", "image": map[string]any{"file_id": "file-1"}}, "image must be a URL; file_id is not supported"},
|
||||
{"reference image limit", "createGeneration", map[string]any{"model": "grok-imagine-video-1.5", "prompt": "x", "reference_images": []any{"a", "b", "c", "d", "e", "f", "g", "h"}}, "reference_images accepts at most 7 items"},
|
||||
{"text-to-video needs a prompt", "createGeneration", map[string]any{"model": "grok-imagine-video-1.5"}, "prompt is required"},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
_, err := callXAI(t, plugin, []string{"native", tc.decoder}, map[string]any{"body": jsonBody(tc.body)})
|
||||
require.ErrorContains(t, err, tc.wantErr)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Clients see why a charged task has no video, and a downstream gateway
|
||||
// running this plugin reads the native status as charged too.
|
||||
func TestXAIModeratedTaskPresentation(t *testing.T) {
|
||||
plugin := newXAIPlugin(t)
|
||||
billedFailure := map[string]any{
|
||||
"status": "failed", "error": map[string]any{"code": "invalid_argument", "message": "Content blocked by moderation"},
|
||||
"usage": map[string]any{"cost_in_usd_ticks": 500000000},
|
||||
}
|
||||
status, err := callXAI(t, plugin, []string{"native", "videoStatus"}, map[string]any{}, map[string]any{"task_id": "task_public", "status": "SUCCESS", "data": billedFailure})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{
|
||||
"status": "done", "progress": float64(100), "video": map[string]any{"respect_moderation": false},
|
||||
"error": map[string]any{"code": "invalid_argument", "message": "Content blocked by moderation"},
|
||||
}, status, "the upstream cost is not exposed")
|
||||
downstream, err := callXAI(t, plugin, []string{"parseTaskResult"}, map[string]any{}, status, map[string]any{"status": 200})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "SUCCESS", downstream["status"])
|
||||
facts, err := callXAI(t, plugin, []string{"extractUsageOnComplete"}, map[string]any{"action": "text_to_video"}, downstream, status)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"moderated": true}, facts)
|
||||
|
||||
stalled, err := callXAI(t, plugin, []string{"native", "videoStatus"}, map[string]any{}, map[string]any{"task_id": "task_public", "status": "FAILURE", "fail_reason": "poll failures", "data": map[string]any{"status": "pending", "progress": 10}})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"status": "failed", "error": map[string]any{"code": "internal_error", "message": "poll failures"}}, stalled)
|
||||
|
||||
filtered := map[string]any{"task_id": "task_public", "status": "SUCCESS", "data": map[string]any{"status": "done", "video": map[string]any{"url": "", "duration": 8, "respect_moderation": false}}}
|
||||
video, err := callXAI(t, plugin, []string{"protocols", "openai_video", "render"}, map[string]any{}, filtered)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{
|
||||
"object": "video", "seconds": "8",
|
||||
"error": map[string]any{"code": "moderation_blocked", "message": "The generated video was blocked by content moderation."},
|
||||
}, video)
|
||||
final, err := callXAI(t, plugin, []string{"protocols", "openai_responses", "renderFinal"}, map[string]any{}, filtered)
|
||||
require.NoError(t, err)
|
||||
encoded, err := common.Marshal(final["output"])
|
||||
require.NoError(t, err)
|
||||
assert.Contains(t, string(encoded), "blocked by content moderation", "renderFinal explains the missing video instead of failing on the absent artifact")
|
||||
}
|
||||
|
||||
// An edit or extension keeps its source's resolution, which xAI never
|
||||
// reports, so the plugin reserves the 720p cap and reads the output file's
|
||||
// MP4 header to settle on the resolution xAI bills.
|
||||
func TestXAIEditSettlesOnOutputResolution(t *testing.T) {
|
||||
service.InitHttpClient()
|
||||
plugin := newXAIPlugin(t)
|
||||
const videoModel = "grok-imagine-video"
|
||||
box := func(kind string, payload ...[]byte) []byte {
|
||||
body := bytes.Join(payload, nil)
|
||||
header := make([]byte, 8)
|
||||
binary.BigEndian.PutUint32(header, uint32(8+len(body)))
|
||||
copy(header[4:], kind)
|
||||
return append(header, body...)
|
||||
}
|
||||
tkhd := func(version byte, width, height uint16) []byte {
|
||||
fields := 72
|
||||
if version == 1 {
|
||||
fields = 84
|
||||
}
|
||||
payload := make([]byte, 4+fields+8)
|
||||
payload[0] = version
|
||||
binary.BigEndian.PutUint16(payload[4+fields:], width)
|
||||
binary.BigEndian.PutUint16(payload[4+fields+4:], height)
|
||||
return box("tkhd", payload)
|
||||
}
|
||||
ftyp := box("ftyp", []byte("isom\x00\x00\x02\x00isomiso2mp41"))
|
||||
// moov first, as in a streaming-friendly file; the audio track has no size.
|
||||
faststart := bytes.Join([][]byte{ftyp, box("moov", box("mvhd", make([]byte, 100)), box("trak", tkhd(0, 0, 0)), box("trak", tkhd(1, 1280, 720))), box("mdat", make([]byte, 1000))}, nil)
|
||||
// moov after a media box longer than the first read.
|
||||
mdat := box("mdat", make([]byte, 300<<10))
|
||||
tail := bytes.Join([][]byte{ftyp, mdat, box("moov", box("trak", tkhd(0, 854, 480)))}, nil)
|
||||
var ranges []string
|
||||
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/faststart.mp4": // ignores Range
|
||||
_, _ = w.Write(faststart)
|
||||
case "/tail.mp4":
|
||||
ranges = append(ranges, r.Header.Get("Range"))
|
||||
http.ServeContent(w, r, "tail.mp4", time.Time{}, bytes.NewReader(tail))
|
||||
case "/text.mp4":
|
||||
_, _ = w.Write([]byte("not a video"))
|
||||
default:
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
}
|
||||
}))
|
||||
defer server.Close()
|
||||
|
||||
const expression = `u("resolution") == "720p" ? tier("720p", u("seconds") * 0.07 + u("input_video_seconds") * 0.01) : ` +
|
||||
`tier("480p", u("seconds") * 0.05 + u("input_video_seconds") * 0.01)`
|
||||
schema, _ := plugin.Meta.UsageForModel(videoModel)
|
||||
require.NoError(t, billing_setting.SmokeTestTaskExpr(expression, schema))
|
||||
info := &relaycommon.RelayInfo{
|
||||
ChannelMeta: &relaycommon.ChannelMeta{ChannelBaseUrl: server.URL, ApiKey: "xai-key", UpstreamModelName: videoModel},
|
||||
OriginModelName: videoModel,
|
||||
TaskRelayInfo: &relaycommon.TaskRelayInfo{PublicTaskID: "task_public", Action: "video_extension"},
|
||||
}
|
||||
adaptor := taskplugin.New(plugin)
|
||||
adaptor.Init(info)
|
||||
c, _ := gin.CreateTestContext(httptest.NewRecorder())
|
||||
c.Request = httptest.NewRequest(http.MethodPost, "/xai/v1/videos/extensions", nil)
|
||||
c.Set("task_request", map[string]any{"model": videoModel, "prompt": "keep walking", "video": map[string]any{"url": "https://cdn.example/in.mp4"}, "duration": 6})
|
||||
require.Nil(t, adaptor.ValidateRequestAndSetAction(c, info))
|
||||
reserved, err := adaptor.ExtractUsageFactsValidated(c, info)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, map[string]any{"seconds": float64(6), "resolution": "720p", "input_images": float64(0), "input_video_seconds": float64(15), "moderated": false}, reserved)
|
||||
snapshot := &billingexpr.BillingSnapshot{
|
||||
BillingMode: billing_setting.BillingModeTieredExpr, ModelName: videoModel, ExprString: expression, ExprHash: billingexpr.ExprHashString(expression),
|
||||
GroupRatio: 1, QuotaPerUnit: 500000, ExprVersion: billingexpr.ExprVersion(expression), TaskUsageBilling: true, UsageFacts: reserved,
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
path string
|
||||
wantResolution string
|
||||
wantTier string
|
||||
wantQuota int
|
||||
}{
|
||||
{name: "a file served whole settles on its 720p track", path: "/faststart.mp4", wantResolution: "720p", wantTier: "720p", wantQuota: 250000},
|
||||
{name: "a moov after the media is read by Range", path: "/tail.mp4", wantResolution: "480p", wantTier: "480p", wantQuota: 190000},
|
||||
{name: "a file that is not MP4 keeps the reserved cap", path: "/text.mp4", wantTier: "720p", wantQuota: 250000},
|
||||
{name: "an unreadable file keeps the reserved cap", path: "/missing.mp4", wantTier: "720p", wantQuota: 250000},
|
||||
} {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
body := `{"status":"done","progress":100,"video":{"url":"` + server.URL + tc.path + `","duration":14,"respect_moderation":true}}`
|
||||
task := &model.Task{
|
||||
TaskID: "task_public", Action: "video_extension", Data: []byte(body),
|
||||
Properties: model.Properties{OriginModelName: videoModel, UpstreamModelName: videoModel},
|
||||
PrivateData: model.TaskPrivateData{PluginState: []byte(`{"extension_seconds":6}`)},
|
||||
}
|
||||
result, err := adaptor.ParseTaskResult(task, &http.Response{StatusCode: http.StatusOK, Header: make(http.Header)}, []byte(body))
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "SUCCESS", result.Status)
|
||||
wantFacts := map[string]any{"input_video_seconds": float64(8)}
|
||||
if tc.wantResolution == "" {
|
||||
assert.Empty(t, result.PluginState)
|
||||
} else {
|
||||
wantFacts["resolution"] = tc.wantResolution
|
||||
assert.JSONEq(t, `{"extension_seconds":6,"output_resolution":"`+tc.wantResolution+`"}`, string(result.PluginState))
|
||||
}
|
||||
assert.Equal(t, wantFacts, result.UsageFacts)
|
||||
settled, _, err := service.EvaluateTaskCompletionUsage(snapshot, result.UsageFacts)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, tc.wantTier, settled.MatchedTier)
|
||||
assert.Equal(t, tc.wantQuota, settled.ActualQuotaAfterGroup)
|
||||
})
|
||||
}
|
||||
moov := len(ftyp) + len(mdat)
|
||||
assert.Equal(t, []string{"bytes=0-65535", fmt.Sprintf("bytes=%d-%d", moov, moov+65535)}, ranges, "the file is read in pieces, never whole")
|
||||
}
|
||||
Reference in new issue
Block a user