74 lines
3.0 KiB
Go
74 lines
3.0 KiB
Go
package model_benchmark
|
|
|
|
import (
|
|
"os"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/QuantumNous/new-api/common"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/require"
|
|
)
|
|
|
|
func TestFingerprintMeowParity(t *testing.T) {
|
|
data, err := os.ReadFile("testdata/meow-reference.json")
|
|
require.NoError(t, err)
|
|
var fixtures []struct {
|
|
Name string `json:"name"`
|
|
Samples []Sample `json:"samples"`
|
|
Expected struct {
|
|
Verdict string `json:"verdict"`
|
|
Matches map[string]float64 `json:"matches"`
|
|
Valid int `json:"valid_samples"`
|
|
} `json:"expected"`
|
|
}
|
|
require.NoError(t, common.Unmarshal(data, &fixtures))
|
|
for _, f := range fixtures {
|
|
t.Run(f.Name, func(t *testing.T) {
|
|
result, err := Score(f.Samples)
|
|
require.NoError(t, err)
|
|
status := f.Expected.Verdict
|
|
if status == "insufficient" {
|
|
status = "inconclusive"
|
|
}
|
|
assert.Equal(t, status, result.Status)
|
|
assert.Equal(t, f.Expected.Valid, result.ValidSamples)
|
|
for model, score := range f.Expected.Matches {
|
|
assert.InDelta(t, score, result.Matches[model], 1e-10, model)
|
|
}
|
|
})
|
|
}
|
|
_, err = Score([]Sample{{Text: "42", Count: 0}, {Text: "42", Count: 0}})
|
|
require.Error(t, err)
|
|
assert.False(t, ValidSample(" "))
|
|
assert.False(t, ValidSample(strings.Repeat("a", 4097)))
|
|
}
|
|
|
|
func TestFingerprintResponseCompletionAndTools(t *testing.T) {
|
|
stream := "data: {\"type\":\"response.output_text.delta\",\"delta\":\"42\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":1}}}\n\n"
|
|
answer, err := ReadResponse(strings.NewReader(stream), "text/event-stream")
|
|
require.NoError(t, err)
|
|
assert.Equal(t, "42", answer.Text)
|
|
assert.True(t, answer.HasUsage)
|
|
answer, err = ReadResponse(strings.NewReader(strings.Replace(stream, `"status":"completed"`, `"status":"completed","output":[]`, 1)), "text/event-stream")
|
|
require.NoError(t, err)
|
|
assert.Empty(t, answer.Text)
|
|
completed := `{"status":"completed","model":"gpt-6-astra","output":[{"type":"message","content":[{"type":"output_text","text":"[1,2,3]"}]}],"usage":{"input_tokens":21,"output_tokens":12,"input_tokens_details":{"cached_tokens":10}}}`
|
|
for _, input := range []struct{ body, content string }{{completed, "application/json"}, {"data: {\"type\":\"response.completed\",\"response\":" + completed + "}\n\n", "text/event-stream"}} {
|
|
r, err := ReadResponse(strings.NewReader(input.body), input.content)
|
|
if err != nil || r.Text != "[1,2,3]" || r.CachedTokens != 10 || !r.HasUsage {
|
|
t.Fatalf("%+v %v", r, err)
|
|
}
|
|
}
|
|
for _, body := range []string{
|
|
"data: {\"type\":\"response.output_text.delta\",\"delta\":\"[1,2,3]\"}\n\n",
|
|
"data: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\"}}\n\n",
|
|
strings.Replace(completed, "completed", "incomplete", 1),
|
|
strings.Replace(completed, "\"type\":\"message\"", "\"type\":\"function_call\"", 1),
|
|
} {
|
|
if _, err := ReadResponse(strings.NewReader(body), "application/json"); err == nil {
|
|
t.Fatal("accepted tool or incomplete sample")
|
|
}
|
|
}
|
|
}
|