feat: sync deployed v1.0.0-rc.38-custom.67

pkg/model_benchmark/fingerprint_test.go

@@ -0,0 +1,73 @@
+package model_benchmark
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/QuantumNous/new-api/common"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFingerprintMeowParity(t *testing.T) {
+	data, err := os.ReadFile("testdata/meow-reference.json")
+	require.NoError(t, err)
+	var fixtures []struct {
+		Name     string   `json:"name"`
+		Samples  []Sample `json:"samples"`
+		Expected struct {
+			Verdict string             `json:"verdict"`
+			Matches map[string]float64 `json:"matches"`
+			Valid   int                `json:"valid_samples"`
+		} `json:"expected"`
+	}
+	require.NoError(t, common.Unmarshal(data, &fixtures))
+	for _, f := range fixtures {
+		t.Run(f.Name, func(t *testing.T) {
+			result, err := Score(f.Samples)
+			require.NoError(t, err)
+			status := f.Expected.Verdict
+			if status == "insufficient" {
+				status = "inconclusive"
+			}
+			assert.Equal(t, status, result.Status)
+			assert.Equal(t, f.Expected.Valid, result.ValidSamples)
+			for model, score := range f.Expected.Matches {
+				assert.InDelta(t, score, result.Matches[model], 1e-10, model)
+			}
+		})
+	}
+	_, err = Score([]Sample{{Text: "42", Count: 0}, {Text: "42", Count: 0}})
+	require.Error(t, err)
+	assert.False(t, ValidSample("   "))
+	assert.False(t, ValidSample(strings.Repeat("a", 4097)))
+}
+
+func TestFingerprintResponseCompletionAndTools(t *testing.T) {
+	stream := "data: {\"type\":\"response.output_text.delta\",\"delta\":\"42\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":1}}}\n\n"
+	answer, err := ReadResponse(strings.NewReader(stream), "text/event-stream")
+	require.NoError(t, err)
+	assert.Equal(t, "42", answer.Text)
+	assert.True(t, answer.HasUsage)
+	answer, err = ReadResponse(strings.NewReader(strings.Replace(stream, `"status":"completed"`, `"status":"completed","output":[]`, 1)), "text/event-stream")
+	require.NoError(t, err)
+	assert.Empty(t, answer.Text)
+	completed := `{"status":"completed","model":"gpt-6-astra","output":[{"type":"message","content":[{"type":"output_text","text":"[1,2,3]"}]}],"usage":{"input_tokens":21,"output_tokens":12,"input_tokens_details":{"cached_tokens":10}}}`
+	for _, input := range []struct{ body, content string }{{completed, "application/json"}, {"data: {\"type\":\"response.completed\",\"response\":" + completed + "}\n\n", "text/event-stream"}} {
+		r, err := ReadResponse(strings.NewReader(input.body), input.content)
+		if err != nil || r.Text != "[1,2,3]" || r.CachedTokens != 10 || !r.HasUsage {
+			t.Fatalf("%+v %v", r, err)
+		}
+	}
+	for _, body := range []string{
+		"data: {\"type\":\"response.output_text.delta\",\"delta\":\"[1,2,3]\"}\n\n",
+		"data: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\"}}\n\n",
+		strings.Replace(completed, "completed", "incomplete", 1),
+		strings.Replace(completed, "\"type\":\"message\"", "\"type\":\"function_call\"", 1),
+	} {
+		if _, err := ReadResponse(strings.NewReader(body), "application/json"); err == nil {
+			t.Fatal("accepted tool or incomplete sample")
+		}
+	}
+}