72 lines
3.3 KiB
Go
72 lines
3.3 KiB
Go
package model_fingerprint
|
|
|
|
import (
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/QuantumNous/new-api/common"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestModelTraceUpstreamParity(t *testing.T) {
|
|
data, err := os.ReadFile("testdata/modeltrace-reference.json")
|
|
require.NoError(t, err)
|
|
var fixtures []struct {
|
|
Samples []Sample
|
|
Expected struct {
|
|
Results []Candidate `json:"results"`
|
|
Valid int `json:"used_outputs"`
|
|
}
|
|
}
|
|
require.NoError(t, common.Unmarshal(data, &fixtures))
|
|
for _, fixture := range fixtures {
|
|
result, err := Score(fixture.Samples)
|
|
require.NoError(t, err)
|
|
assert.Equal(t, Version, result.Engine)
|
|
assert.Equal(t, fixture.Expected.Valid, result.ValidSamples)
|
|
assert.Equal(t, fixture.Expected.Results[0].Model, result.Closest)
|
|
for _, candidate := range fixture.Expected.Results {
|
|
assert.InDelta(t, candidate.Weight, result.Matches[candidate.Model], 1e-9)
|
|
}
|
|
}
|
|
assert.False(t, ValidSample("I cannot provide the sequence."))
|
|
assert.True(t, ValidArray(strings.Repeat("42,", 180), 300))
|
|
assert.False(t, ValidArray(strings.Repeat("42,", 100), 300))
|
|
result := Decide([]Candidate{{Model: Model, Weight: .79, Score: 2}, {Model: "other", Weight: .21, Score: 1}}, 3)
|
|
assert.Equal(t, "inconclusive", result.Status)
|
|
result = Decide([]Candidate{{Model: "other", Weight: .95, Score: 2}, {Model: Model, Weight: .05, Score: 1}}, 2)
|
|
assert.Equal(t, "inconclusive", result.Status)
|
|
result = Decide([]Candidate{{Model: "other", Weight: .95, Score: 2}, {Model: Model, Weight: .05, Score: 1}}, 3)
|
|
assert.Equal(t, "mismatch", result.Status)
|
|
}
|
|
|
|
func TestFingerprintResponseCompletionAndTools(t *testing.T) {
|
|
stream := "data: {\"type\":\"response.output_text.delta\",\"delta\":\"42\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":1}}}\n\n"
|
|
answer, err := ReadResponse(strings.NewReader(stream), "text/event-stream")
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "42", answer.Text)
|
|
assert.True(t, answer.HasUsage)
|
|
answer, err = ReadResponse(strings.NewReader(strings.Replace(stream, `"status":"completed"`, `"status":"completed","output":[]`, 1)), "text/event-stream")
|
|
require.NoError(t, err)
|
|
assert.Empty(t, answer.Text)
|
|
completed := `{"status":"completed","model":"gpt-6-astra","output":[{"type":"message","content":[{"type":"output_text","text":"[1,2,3]"}]}],"usage":{"input_tokens":21,"output_tokens":12,"input_tokens_details":{"cached_tokens":10}}}`
|
|
for _, input := range []struct{ body, content string }{{completed, "application/json"}, {"data: {\"type\":\"response.completed\",\"response\":" + completed + "}\n\n", "text/event-stream"}} {
|
|
r, err := ReadResponse(strings.NewReader(input.body), input.content)
|
|
if err != nil || r.Text != "[1,2,3]" || r.CachedTokens != 10 || !r.HasUsage {
|
|
t.Fatalf("%+v %v", r, err)
|
|
}
|
|
}
|
|
for _, body := range []string{
|
|
"data: {\"type\":\"response.output_text.delta\",\"delta\":\"[1,2,3]\"}\n\n",
|
|
"data: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\"}}\n\n",
|
|
strings.Replace(completed, "completed", "incomplete", 1),
|
|
strings.Replace(completed, "\"type\":\"message\"", "\"type\":\"function_call\"", 1),
|
|
} {
|
|
if _, err := ReadResponse(strings.NewReader(body), "application/json"); err == nil {
|
|
t.Fatal("accepted tool or incomplete sample")
|
|
}
|
|
}
|
|
}
|