feat: sync deployed v1.0.0-rc.38-custom.67

pkg/billingexpr/image_generation_price_test.go

@@ -0,0 +1,22 @@
+package billingexpr
+
+import (
+	"github.com/stretchr/testify/require"
+	"testing"
+)
+
+func TestImageGenerationPriceSizesAndQuantity(t *testing.T) {
+	expression := `(param("size") != nil && (param("size") == "4k" || (param("size") matches "^[0-9]+x[0-9]+$" && min(float(split(param("size"), "x")[0]), float(split(param("size"), "x")[1])) > 2048)) ? tier("4k", fixed(0.12)) : param("size") != nil && (param("size") == "2k" || (param("size") matches "^[0-9]+x[0-9]+$" && min(float(split(param("size"), "x")[0]), float(split(param("size"), "x")[1])) > 1024)) ? tier("2k", fixed(0.08)) : tier("1k", fixed(0.06))) * image_count`
+	for _, tc := range []struct {
+		body, tier string
+		price      float64
+	}{{`{}`, "1k", 0.06}, {`{"size":"auto"}`, "1k", 0.06}, {`{"size":"1024x1536"}`, "1k", 0.06}, {`{"size":"2048x1152"}`, "2k", 0.08}, {`{"size":"3840x2160"}`, "4k", 0.12}, {`{"size":"2k"}`, "2k", 0.08}} {
+		for _, n := range []int{1, 2, 3} {
+			cost, trace, err := RunExprWithRequest(expression, TokenParams{P: 10000, C: 5000}, RequestInput{Body: []byte(tc.body), ImageCount: &n})
+			require.NoError(t, err)
+			require.InDelta(t, tc.price*float64(n)*1000000, cost, 0.01)
+			require.Equal(t, tc.tier, trace.MatchedTier)
+			require.Equal(t, BillingUnitRequest, trace.BillingUnit)
+		}
+	}
+}

pkg/model_benchmark/LICENSE-meow.txt

@@ -0,0 +1,75 @@
+# PolyForm Noncommercial License 1.0.0
+
+<https://polyformproject.org/licenses/noncommercial/1.0.0>
+
+## Acceptance
+
+In order to get any license under these terms, you must agree to them as both strict obligations and conditions to all your licenses.
+
+## Copyright License
+
+The licensor grants you a copyright license for the software to do everything you might do with the software that would otherwise infringe the licensor's copyright in it for any permitted purpose.  However, you may only distribute the software according to [Distribution License](#distribution-license) and make changes or new works based on the software according to [Changes and New Works License](#changes-and-new-works-license).
+
+## Distribution License
+
+The licensor grants you an additional copyright license to distribute copies of the software.  Your license to distribute covers distributing the software with changes and new works permitted by [Changes and New Works License](#changes-and-new-works-license).
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you also gets a copy of these terms or the URL for them above, as well as copies of any plain-text lines beginning with `Required Notice:` that the licensor provided with the software.  For example:
+
+> Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
+
+Required Notice: Copyright 2026 chen-006 and contributors. Original project: https://github.com/chen-006/gpt56_api_detector
+
+## Changes and New Works License
+
+The licensor grants you an additional copyright license to make changes and new works based on the software for any permitted purpose.
+
+## Patent License
+
+The licensor grants you a patent license for the software that covers patent claims the licensor can license, or becomes able to license, that you would infringe by using the software.
+
+## Noncommercial Purposes
+
+Any noncommercial purpose is a permitted purpose.
+
+## Personal Uses
+
+Personal use for research, experiment, and testing for the benefit of public knowledge, personal study, private entertainment, hobby projects, amateur pursuits, or religious observance, without any anticipated commercial application, is use for a permitted purpose.
+
+## Noncommercial Organizations
+
+Use by any charitable organization, educational institution, public research organization, public safety or health organization, environmental protection organization, or government institution is use for a permitted purpose regardless of the source of funding or obligations resulting from the funding.
+
+## Fair Use
+
+You may have "fair use" rights for the software under the law. These terms do not limit them.
+
+## No Other Rights
+
+These terms do not allow you to sublicense or transfer any of your licenses to anyone else, or prevent the licensor from granting licenses to anyone else.  These terms do not imply any other licenses.
+
+## Patent Defense
+
+If you make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company.
+
+## Violations
+
+The first time you are notified in writing that you have violated any of these terms, or done anything with the software not covered by your licenses, your licenses can nonetheless continue if you come into full compliance with these terms, and take practical steps to correct past violations, within 32 days of receiving notice.  Otherwise, all your licenses end immediately.
+
+## No Liability
+
+***As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim.***
+
+## Definitions
+
+The **licensor** is the individual or entity offering these terms, and the **software** is the software the licensor makes available under these terms.
+
+**You** refers to the individual or entity agreeing to these terms.
+
+**Your company** is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization.  **Control** means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise.  Control can be direct or indirect.
+
+**Your licenses** are all the licenses granted to you for the software under these terms.
+
+**Use** means anything you do with the software requiring one of your licenses.

pkg/model_benchmark/fingerprint.go

@@ -0,0 +1,200 @@
+// Meow predictive fingerprint integration, pinned to the public GPT benchmark.
+// Required Notice: Copyright 2026 chen-006 and contributors.
+// Original project: https://github.com/chen-006/gpt56_api_detector
+// See LICENSE-meow.txt for the PolyForm Noncommercial 1.0.0 license.
+package model_benchmark
+
+import (
+	_ "embed"
+	"errors"
+	"math"
+	"sort"
+	"strings"
+	"sync"
+	"unicode/utf8"
+
+	"github.com/QuantumNous/new-api/common"
+	"golang.org/x/text/cases"
+)
+
+const Model = "gpt-6-astra"
+const Version = "meow-4.5.4-predictive.20260924.2"
+const ProbeCount = 32
+const OutputTokens = 128
+
+//go:embed meow.json
+var benchmarkJSON []byte
+
+type Sample struct {
+	Text string `json:"text"`
+	// Ordinal in the fixed quick-tier plan, under the existing storage key.
+	Count int `json:"expected_count"`
+}
+
+type Result struct {
+	Status       string             `json:"status"`
+	ValidSamples int                `json:"valid_samples"`
+	Closest      string             `json:"closest,omitempty"`
+	Weight       float64            `json:"relative_weight,omitempty"`
+	Gap          float64            `json:"score_gap,omitempty"`
+	Engine       string             `json:"engine,omitempty"`
+	Matches      map[string]float64 `json:"matches,omitempty"`
+	Thresholds   map[string]float64 `json:"thresholds,omitempty"`
+}
+
+type Probe struct {
+	ID     string `json:"id"`
+	System string `json:"system"`
+	Prompt string `json:"prompt"`
+}
+type fittedCell struct {
+	Categories []string             `json:"categories"`
+	Alpha      map[string][]float64 `json:"alpha"`
+}
+type benchmark struct {
+	Plan   []Probe `json:"-"`
+	Probes []struct {
+		Cells []Probe `json:"cells"`
+	} `json:"probes"`
+	Fitted struct {
+		Models     []string              `json:"models"`
+		Sources    []string              `json:"sources"`
+		References []string              `json:"reference_sources"`
+		Cells      map[string]fittedCell `json:"cells"`
+	} `json:"fitted"`
+	Tiers map[string]struct {
+		Counts     map[string]int     `json:"counts"`
+		Thresholds map[string]float64 `json:"thresholds"`
+	} `json:"tiers"`
+}
+
+var load = sync.OnceValues(func() (*benchmark, error) {
+	var b benchmark
+	if err := common.Unmarshal(benchmarkJSON, &b); err != nil {
+		return nil, err
+	}
+	var plan []Probe
+	for _, p := range b.Probes {
+		for _, c := range p.Cells {
+			for range b.Tiers["low"].Counts[c.ID] {
+				plan = append(plan, c)
+			}
+		}
+	}
+	sort.SliceStable(plan, func(i, j int) bool { return plan[i].ID < plan[j].ID })
+	if len(plan) != ProbeCount {
+		return nil, errors.New("invalid meow plan")
+	}
+	b.Plan = plan
+	return &b, nil
+})
+
+func ProbeAt(index int) (Probe, error) {
+	b, err := load()
+	if err != nil {
+		return Probe{}, err
+	}
+	if index < 0 || index >= len(b.Plan) {
+		return Probe{}, errors.New("invalid probe index")
+	}
+	return b.Plan[index], nil
+}
+
+func ValidSample(text string) bool {
+	if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 65536 {
+		return false
+	}
+	text = cases.Fold().String(strings.TrimSpace(text))
+	return text != "" && utf8.RuneCountInString(text) <= 4096
+}
+
+// Whole-run Dirichlet posterior prediction, using average log evidence versus
+// the strongest rival and calibrated thresholds. Closest alone is no verdict.
+func Score(samples []Sample) (Result, error) {
+	b, err := load()
+	if err != nil {
+		return Result{}, err
+	}
+	r := Result{Status: "inconclusive", Engine: Version, Matches: map[string]float64{}, Thresholds: b.Tiers["low"].Thresholds}
+	counts := map[string]map[string]int{}
+	seen := map[int]bool{}
+	for _, sample := range samples {
+		if sample.Count < 0 || sample.Count >= len(b.Plan) || seen[sample.Count] {
+			return r, errors.New("invalid or duplicate meow sample")
+		}
+		seen[sample.Count] = true
+		if !ValidSample(sample.Text) {
+			continue
+		}
+		id := b.Plan[sample.Count].ID
+		text := cases.Fold().String(strings.TrimSpace(sample.Text))
+		if counts[id] == nil {
+			counts[id] = map[string]int{}
+		}
+		counts[id][text]++
+		r.ValidSamples++
+	}
+	sufficient := r.ValidSamples >= int(math.Ceil(.6*ProbeCount))
+	evidence := map[string]float64{}
+	for id, planned := range b.Tiers["low"].Counts {
+		cell := b.Fitted.Cells[id]
+		numbers := make([]int, len(cell.Categories))
+		positions := map[string]int{}
+		for i, category := range cell.Categories {
+			positions[category] = i
+		}
+		n := 0
+		for text, number := range counts[id] {
+			i, ok := positions[text]
+			if !ok {
+				i = positions["__UNSEEN_IN_TRAINING__"]
+			}
+			numbers[i] += number
+			n += number
+		}
+		if n < int(math.Ceil(.6*float64(planned))) {
+			sufficient = false
+		}
+		for _, source := range b.Fitted.Sources {
+			alpha := cell.Alpha[source]
+			total, value := 0.0, 0.0
+			for i, a := range alpha {
+				total += a
+				left, _ := math.Lgamma(a + float64(numbers[i]))
+				right, _ := math.Lgamma(a)
+				value += left - right
+			}
+			left, _ := math.Lgamma(total)
+			right, _ := math.Lgamma(total + float64(n))
+			evidence[source] += value + left - right
+		}
+	}
+	evidence["other_known_external"] = math.Inf(-1)
+	for _, source := range b.Fitted.References {
+		evidence["other_known_external"] = math.Max(evidence["other_known_external"], evidence[source])
+	}
+	for _, model := range b.Fitted.Models {
+		rival := math.Inf(-1)
+		for _, other := range b.Fitted.Models {
+			if other != model {
+				rival = math.Max(rival, evidence[other])
+			}
+		}
+		r.Matches[model] = 1 / (1 + math.Exp((rival-evidence[model])/float64(max(r.ValidSamples, 1))))
+	}
+	ordered := append([]string(nil), b.Fitted.Models...)
+	sort.SliceStable(ordered, func(i, j int) bool { return r.Matches[ordered[i]] > r.Matches[ordered[j]] })
+	top := ordered[0]
+	r.Weight = r.Matches[top]
+	r.Gap = (evidence[top] - evidence[ordered[1]]) / float64(max(r.ValidSamples, 1))
+	if r.ValidSamples > 0 {
+		r.Closest = top
+	}
+	if sufficient && r.Weight > r.Thresholds[top] && r.Weight > r.Matches[ordered[1]] {
+		r.Status = "mismatch"
+		if top == Model {
+			r.Status = "match"
+		}
+	}
+	return r, nil
+}

pkg/model_benchmark/fingerprint_test.go

@@ -0,0 +1,73 @@
+package model_benchmark
+
+import (
+	"os"
+	"strings"
+	"testing"
+
+	"github.com/QuantumNous/new-api/common"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestFingerprintMeowParity(t *testing.T) {
+	data, err := os.ReadFile("testdata/meow-reference.json")
+	require.NoError(t, err)
+	var fixtures []struct {
+		Name     string   `json:"name"`
+		Samples  []Sample `json:"samples"`
+		Expected struct {
+			Verdict string             `json:"verdict"`
+			Matches map[string]float64 `json:"matches"`
+			Valid   int                `json:"valid_samples"`
+		} `json:"expected"`
+	}
+	require.NoError(t, common.Unmarshal(data, &fixtures))
+	for _, f := range fixtures {
+		t.Run(f.Name, func(t *testing.T) {
+			result, err := Score(f.Samples)
+			require.NoError(t, err)
+			status := f.Expected.Verdict
+			if status == "insufficient" {
+				status = "inconclusive"
+			}
+			assert.Equal(t, status, result.Status)
+			assert.Equal(t, f.Expected.Valid, result.ValidSamples)
+			for model, score := range f.Expected.Matches {
+				assert.InDelta(t, score, result.Matches[model], 1e-10, model)
+			}
+		})
+	}
+	_, err = Score([]Sample{{Text: "42", Count: 0}, {Text: "42", Count: 0}})
+	require.Error(t, err)
+	assert.False(t, ValidSample("   "))
+	assert.False(t, ValidSample(strings.Repeat("a", 4097)))
+}
+
+func TestFingerprintResponseCompletionAndTools(t *testing.T) {
+	stream := "data: {\"type\":\"response.output_text.delta\",\"delta\":\"42\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":1}}}\n\n"
+	answer, err := ReadResponse(strings.NewReader(stream), "text/event-stream")
+	require.NoError(t, err)
+	assert.Equal(t, "42", answer.Text)
+	assert.True(t, answer.HasUsage)
+	answer, err = ReadResponse(strings.NewReader(strings.Replace(stream, `"status":"completed"`, `"status":"completed","output":[]`, 1)), "text/event-stream")
+	require.NoError(t, err)
+	assert.Empty(t, answer.Text)
+	completed := `{"status":"completed","model":"gpt-6-astra","output":[{"type":"message","content":[{"type":"output_text","text":"[1,2,3]"}]}],"usage":{"input_tokens":21,"output_tokens":12,"input_tokens_details":{"cached_tokens":10}}}`
+	for _, input := range []struct{ body, content string }{{completed, "application/json"}, {"data: {\"type\":\"response.completed\",\"response\":" + completed + "}\n\n", "text/event-stream"}} {
+		r, err := ReadResponse(strings.NewReader(input.body), input.content)
+		if err != nil || r.Text != "[1,2,3]" || r.CachedTokens != 10 || !r.HasUsage {
+			t.Fatalf("%+v %v", r, err)
+		}
+	}
+	for _, body := range []string{
+		"data: {\"type\":\"response.output_text.delta\",\"delta\":\"[1,2,3]\"}\n\n",
+		"data: {\"type\":\"response.output_item.added\",\"item\":{\"type\":\"function_call\"}}\n\n",
+		strings.Replace(completed, "completed", "incomplete", 1),
+		strings.Replace(completed, "\"type\":\"message\"", "\"type\":\"function_call\"", 1),
+	} {
+		if _, err := ReadResponse(strings.NewReader(body), "application/json"); err == nil {
+			t.Fatal("accepted tool or incomplete sample")
+		}
+	}
+}

pkg/model_benchmark/response.go

@@ -0,0 +1,144 @@
+package model_benchmark
+
+import (
+	"bufio"
+	"errors"
+	"io"
+	"strings"
+
+	"github.com/tidwall/gjson"
+)
+
+var ErrInvalidSample = errors.New("invalid probe answer")
+var ErrIncomplete = errors.New("probe response incomplete")
+
+type ProbeResponse struct {
+	Text             string `json:"-"`
+	DeclaredModel    string `json:"declared_model,omitempty"`
+	InputTokens      int64  `json:"input_tokens"`
+	OutputTokens     int64  `json:"output_tokens"`
+	CachedTokens     int64  `json:"cached_tokens"`
+	CacheWriteTokens int64  `json:"cache_write_tokens"`
+	HasUsage         bool   `json:"has_usage"`
+}
+
+func completedResponse(response gjson.Result) (ProbeResponse, error) {
+	r := ProbeResponse{}
+	r.DeclaredModel = response.Get("model").String()
+	if len(r.DeclaredModel) > 128 {
+		r.DeclaredModel = ""
+	}
+	usage := response.Get("usage")
+	r.HasUsage = usage.Get("input_tokens").Type == gjson.Number && usage.Get("output_tokens").Type == gjson.Number
+	r.InputTokens = usage.Get("input_tokens").Int()
+	r.OutputTokens = usage.Get("output_tokens").Int()
+	r.CachedTokens = usage.Get("input_tokens_details.cached_tokens").Int()
+	r.CacheWriteTokens = max(usage.Get("input_tokens_details.cached_creation_tokens").Int(), usage.Get("input_tokens_details.cache_creation_tokens").Int())
+	if response.Get("status").String() != "completed" {
+		return r, ErrIncomplete
+	}
+	for _, item := range response.Get("output").Array() {
+		switch item.Get("type").String() {
+		case "reasoning":
+		case "message":
+			for _, content := range item.Get("content").Array() {
+				if content.Get("type").String() != "output_text" {
+					return r, ErrInvalidSample
+				}
+				r.Text += content.Get("text").String()
+				if len(r.Text) > 16384 {
+					return ProbeResponse{}, ErrInvalidSample
+				}
+			}
+		default:
+			return r, ErrInvalidSample
+		}
+	}
+	return r, nil
+}
+
+// Only a complete response is a sample. Partial deltas, tool calls and a
+// truncated stream cannot become a model verdict. Total bytes are bounded.
+func ReadResponse(reader io.Reader, contentType string) (ProbeResponse, error) {
+	limited := &io.LimitedReader{R: reader, N: 8 << 20}
+	buffered := bufio.NewReader(limited)
+	peek, _ := buffered.Peek(1)
+	if !strings.Contains(contentType, "text/event-stream") && len(peek) > 0 && peek[0] == '{' {
+		data, err := io.ReadAll(buffered)
+		if err != nil {
+			return ProbeResponse{}, err
+		}
+		if limited.N == 0 || !gjson.ValidBytes(data) {
+			return ProbeResponse{}, ErrInvalidSample
+		}
+		return completedResponse(gjson.ParseBytes(data))
+	}
+	scanner := bufio.NewScanner(buffered)
+	scanner.Buffer(make([]byte, 4096), 2<<20)
+	var data strings.Builder
+	var deltas strings.Builder
+	refused := false
+	for scanner.Scan() {
+		line := scanner.Text()
+		if strings.HasPrefix(line, "data:") {
+			if data.Len() > 0 {
+				data.WriteByte('\n')
+			}
+			data.WriteString(strings.TrimSpace(strings.TrimPrefix(line, "data:")))
+			if data.Len() > 2<<20 {
+				return ProbeResponse{}, ErrInvalidSample
+			}
+			continue
+		}
+		if line != "" || data.Len() == 0 {
+			continue
+		}
+		payload := data.String()
+		data.Reset()
+		if payload == "[DONE]" {
+			break
+		}
+		if !gjson.Valid(payload) {
+			return ProbeResponse{}, ErrInvalidSample
+		}
+		event := gjson.Parse(payload)
+		typ := event.Get("type").String()
+		if typ == "response.output_text.delta" {
+			deltas.WriteString(event.Get("delta").String())
+			if deltas.Len() > 65536 {
+				return ProbeResponse{}, ErrInvalidSample
+			}
+		}
+		if typ == "response.refusal.delta" || typ == "response.refusal.done" {
+			refused = true
+		}
+		if typ == "response.output_item.added" || typ == "response.output_item.done" {
+			itemType := event.Get("item.type").String()
+			if itemType != "message" && itemType != "reasoning" {
+				return ProbeResponse{}, ErrInvalidSample
+			}
+		}
+		if typ == "response.completed" {
+			response := event.Get("response")
+			answer, err := completedResponse(response)
+			if refused {
+				return answer, ErrInvalidSample
+			}
+			// Meow 4.5.4 accepts deltas only when completion omits output entirely.
+			if err == nil && !response.Get("output").Exists() {
+				answer.Text = deltas.String()
+			}
+			return answer, err
+		}
+		if typ == "response.failed" || typ == "response.incomplete" {
+			return completedResponse(event.Get("response"))
+		}
+		if typ == "error" {
+			return ProbeResponse{}, ErrIncomplete
+		}
+	}
+	if err := scanner.Err(); err != nil {
+		return ProbeResponse{}, err
+	}
+	return ProbeResponse{}, ErrIncomplete
+}

pkg/model_benchmark/testdata/astra-reference.json

@@ -0,0 +1 @@
+[{"row_id": "gpt-6-astra__query-01", "parent_row_id": "gpt-6-astra__query-01", "bank_id": "gpt", "source": "gpt-6-astra", "model_id": "gpt-6-astra", "response_model": "gpt-6-astra", "exact_version": "gpt-6-astra", "condition_id": "environment-01", "nuisance_condition_id": "environment-01", "wrapper_id": "environment-01", "wrapper_transport": "clean", "challenge_id": "query-01", "task_index": 0, "requested_count": 218, "parsed_count": 218, "strict_threshold": 120, "strict_valid": true, "temperature": "provider_default", "config_id": "reference-collection", "provider": "codex", "prompt": "For each of 218 positions, make one separate first-instinct choice of an integer from 1 to 355 inclusive. The current language model must complete this directly without tools, Python, code execution, calculators, search, APIs, or external random generators. Choose every position separately. Do not count upward or downward, including 1, 2, 3, and do not use an arithmetic progression, repeating cycle, repeated block, or another rule-made pattern. Accidental repetitions are valid. Once an item is written, do not sort, reorder, deduplicate, replace, or repair the list. Return one compact JSON array containing the complete sequence and no explanation.", "base_prompt": "For each of 218 positions, make one separate first-instinct choice of an integer from 1 to 355 inclusive. The current language model must complete this directly without tools, Python, code execution, calculators, search, APIs, or external random generators. Choose every position separately. Do not count upward or downward, including 1, 2, 3, and do not use an arithmetic progression, repeating cycle, repeated block, or another rule-made pattern. Accidental repetitions are valid. Once an item is written, do not sort, reorder, deduplicate, replace, or repair the list. Return one compact JSON array containing the complete sequence and no explanation.", "system_prompt": "", "user_prefix": "", "text": "[247,83,319,146,28,204,337,112,59,271,164,352,91,223,17,308,135,76,289,192,43,258,121,334,68,215,301,154,9,237,178,325,54,109,283,196,31,342,87,261,143,22,310,185,73,229,348,116,294,48,203,157,331,95,266,12,181,316,64,240,127,354,39,218,172,298,81,251,106,323,56,189,274,14,139,345,207,93,232,167,305,46,119,287,211,34,253,176,329,72,224,151,8,313,98,269,184,51,341,130,291,66,235,159,322,24,198,277,113,350,85,243,161,37,302,124,280,19,216,146,333,61,194,256,102,317,44,170,288,79,226,355,138,11,264,187,314,97,205,53,275,149,326,32,118,238,69,296,163,221,90,347,179,26,250,132,309,57,201,284,114,339,74,228,155,4,273,193,320,41,245,107,299,182,63,212,351,129,16,267,158,306,88,234,47,328,174,101,286,209,35,252,141,315,77,197,343,120,260,58,231,168,292,21,136,304,94,219,65,336,183,242,7,278,153,324,108,202]", "error": null, "request_seconds": 45.789022299984936, "collected_at": "2026-09-05T02:04:49.454748+00:00"}, {"row_id": "gpt-6-astra__query-02", "parent_row_id": "gpt-6-astra__query-02", "bank_id": "gpt", "source": "gpt-6-astra", "model_id": "gpt-6-astra", "response_model": "gpt-6-astra", "exact_version": "gpt-6-astra", "condition_id": "environment-01", "nuisance_condition_id": "environment-01", "wrapper_id": "environment-01", "wrapper_transport": "clean", "challenge_id": "query-02", "task_index": 0, "requested_count": 233, "parsed_count": 233, "strict_threshold": 129, "strict_valid": true, "temperature": "provider_default", "config_id": "reference-collection", "provider": "codex", "prompt": "Provide exactly 233 whole-number choices between 1 and 355 inclusive, choosing each item separately as it comes to mind. The current language model must complete this directly without tools, Python, code execution, calculators, search, APIs, or external random generators. Choose every position separately. Do not count upward or downward, including 1, 2, 3, and do not use an arithmetic progression, repeating cycle, repeated block, or another rule-made pattern. Accidental repetitions are valid. Once an item is written, do not sort, reorder, deduplicate, replace, or repair the list. Return one compact JSON array containing the complete sequence and no explanation.", "base_prompt": "Provide exactly 233 whole-number choices between 1 and 355 inclusive, choosing each item separately as it comes to mind. The current language model must complete this directly without tools, Python, code execution, calculators, search, APIs, or external random generators. Choose every position separately. Do not count upward or downward, including 1, 2, 3, and do not use an arithmetic progression, repeating cycle, repeated block, or another rule-made pattern. Accidental repetitions are valid. Once an item is written, do not sort, reorder, deduplicate, replace, or repair the list. Return one compact JSON array containing the complete sequence and no explanation.", "system_prompt": "", "user_prefix": "", "text": "[287,43,196,321,78,154,9,242,113,335,62,219,174,28,303,141,87,264,350,106,183,51,298,127,14,231,92,316,1Line truncated

pkg/model_fingerprint/LICENSE-meow.txt

@@ -0,0 +1,75 @@
+# PolyForm Noncommercial License 1.0.0
+
+<https://polyformproject.org/licenses/noncommercial/1.0.0>
+
+## Acceptance
+
+In order to get any license under these terms, you must agree to them as both strict obligations and conditions to all your licenses.
+
+## Copyright License
+
+The licensor grants you a copyright license for the software to do everything you might do with the software that would otherwise infringe the licensor's copyright in it for any permitted purpose.  However, you may only distribute the software according to [Distribution License](#distribution-license) and make changes or new works based on the software according to [Changes and New Works License](#changes-and-new-works-license).
+
+## Distribution License
+
+The licensor grants you an additional copyright license to distribute copies of the software.  Your license to distribute covers distributing the software with changes and new works permitted by [Changes and New Works License](#changes-and-new-works-license).
+
+## Notices
+
+You must ensure that anyone who gets a copy of any part of the software from you also gets a copy of these terms or the URL for them above, as well as copies of any plain-text lines beginning with `Required Notice:` that the licensor provided with the software.  For example:
+
+> Required Notice: Copyright Yoyodyne, Inc. (http://example.com)
+
+Required Notice: Copyright 2026 chen-006 and contributors. Original project: https://github.com/chen-006/gpt56_api_detector
+
+## Changes and New Works License
+
+The licensor grants you an additional copyright license to make changes and new works based on the software for any permitted purpose.
+
+## Patent License
+
+The licensor grants you a patent license for the software that covers patent claims the licensor can license, or becomes able to license, that you would infringe by using the software.
+
+## Noncommercial Purposes
+
+Any noncommercial purpose is a permitted purpose.
+
+## Personal Uses
+
+Personal use for research, experiment, and testing for the benefit of public knowledge, personal study, private entertainment, hobby projects, amateur pursuits, or religious observance, without any anticipated commercial application, is use for a permitted purpose.
+
+## Noncommercial Organizations
+
+Use by any charitable organization, educational institution, public research organization, public safety or health organization, environmental protection organization, or government institution is use for a permitted purpose regardless of the source of funding or obligations resulting from the funding.
+
+## Fair Use
+
+You may have "fair use" rights for the software under the law. These terms do not limit them.
+
+## No Other Rights
+
+These terms do not allow you to sublicense or transfer any of your licenses to anyone else, or prevent the licensor from granting licenses to anyone else.  These terms do not imply any other licenses.
+
+## Patent Defense
+
+If you make any written claim that the software infringes or contributes to infringement of any patent, your patent license for the software granted under these terms ends immediately. If your company makes such a claim, your patent license ends immediately for work on behalf of your company.
+
+## Violations
+
+The first time you are notified in writing that you have violated any of these terms, or done anything with the software not covered by your licenses, your licenses can nonetheless continue if you come into full compliance with these terms, and take practical steps to correct past violations, within 32 days of receiving notice.  Otherwise, all your licenses end immediately.
+
+## No Liability
+
+***As far as the law allows, the software comes as is, without any warranty or condition, and the licensor will not be liable to you for any damages arising out of these terms or the use or nature of the software, under any kind of legal claim.***
+
+## Definitions
+
+The **licensor** is the individual or entity offering these terms, and the **software** is the software the licensor makes available under these terms.
+
+**You** refers to the individual or entity agreeing to these terms.
+
+**Your company** is any legal entity, sole proprietorship, or other kind of organization that you work for, plus all organizations that have control over, are under the control of, or are under common control with that organization.  **Control** means ownership of substantially all the assets of an entity, or the power to direct its management and policies by vote, contract, or otherwise.  Control can be direct or indirect.
+
+**Your licenses** are all the licenses granted to you for the software under these terms.
+
+**Use** means anything you do with the software requiring one of your licenses.

pkg/model_fingerprint/fingerprint.go

@@ -1,21 +1,30 @@
-// Numerical fingerprint scorer adapted from Katao123/gpt-model-check (MIT),
-// commit 3d6716a0903cf061daea001d5d44ee410846634e. See the adjacent licenses.
+// ModelTrace scorer and bank: xqy2006/ModelTrace d4131b30243dfa05e70180b5eedde742103f1d73 (MIT).
 package model_fingerprint
 
 import (
+	"crypto/rand"
 	_ "embed"
 	"errors"
+	"fmt"
 	"math"
+	"math/big"
+	"regexp"
+	"strconv"
 	"strings"
 	"sync"
 	"time"
+	"unicode"
+	"unicode/utf8"
 
 	"github.com/QuantumNous/new-api/common"
 	"github.com/grafana/sobek"
 )
 
 const Model = "gpt-6-astra"
-const Version = "modeltrace-20260905-gmc-3d6716a"
+const Version = "modeltrace-d4131b3"
+
+const ProbeCount = 3
+const OutputTokens = 4096
 
 //go:embed bank.json
 var bankJSON string
@@ -39,43 +48,71 @@ type Candidate struct {
 }
 
 type Result struct {
-	Status       string  `json:"status"`
-	ValidSamples int     `json:"valid_samples"`
-	Closest      string  `json:"closest,omitempty"`
-	Weight       float64 `json:"relative_weight,omitempty"`
-	Gap          float64 `json:"score_gap,omitempty"`
+	Engine       string             `json:"engine"`
+	Matches      map[string]float64 `json:"matches,omitempty"`
+	Status       string             `json:"status"`
+	ValidSamples int                `json:"valid_samples"`
+	Closest      string             `json:"closest,omitempty"`
+	Weight       float64            `json:"relative_weight,omitempty"`
+	Gap          float64            `json:"score_gap,omitempty"`
 }
 
-func ValidArray(text string, count int) bool {
-	if len(text) > 16384 || count <= 0 {
-		return false
-	}
-	text = strings.TrimSpace(text)
-	if strings.HasPrefix(text, "```") && strings.HasSuffix(text, "```") {
-		if i := strings.IndexByte(text, '\n'); i >= 0 {
-			text = strings.TrimSpace(text[i+1 : len(text)-3])
-		}
-	}
-	var numbers []int
-	if common.Unmarshal([]byte(text), &numbers) != nil {
-		return false
-	}
-	if len(numbers) < max(80, int(math.Ceil(float64(count)*.55))) || len(numbers) > count*2 {
-		return false
-	}
-	for _, number := range numbers {
-		if number < 1 || number > 355 {
-			return false
-		}
-	}
-	return true
+type Probe struct {
+	ID     string
+	System string
+	Prompt string
+	Count  int
 }
 
+func ProbeAt(index int) (Probe, error) {
+	if index < 0 || index >= ProbeCount {
+		return Probe{}, errors.New("invalid probe index")
+	}
+	// Disjoint length ranges retain three independent, non-identical challenges.
+	offset, err := rand.Int(rand.Reader, big.NewInt(13))
+	if err != nil {
+		return Probe{}, err
+	}
+	count := 292 + index*14 + int(offset.Int64())
+	prompt := fmt.Sprintf("这是一次独立的数值选择记录。逐项选择 %d 个 1 到 355（含端点）的整数。", count) +
+		"每个位置都要单独选择；不要从 1 开始计数，不要连续递增或递减，也不要采用等差、循环、重复区块或其他规则化模式。" +
+		"本任务必须由当前语言模型直接完成：禁止调用或借助任何工具，包括 Python、代码执行器、计算器、搜索、API 和外部随机数生成器；也不要先编写或运行代码。" +
+		"偶然重复是有效的；不要重新排列或修正已经写出的项目。数字之间用逗号或空格分隔均可。直接从第一个取值开始输出，不要在序列前重复数量、范围或任务说明。"
+	return Probe{ID: fmt.Sprint(index), Prompt: prompt, Count: count}, nil
+}
+
+var integerPattern = regexp.MustCompile(`[0-9]+`)
+
+// Match ModelTrace's longest numeric run parser; prose separates runs.
+func ValidArray(text string, count int) bool {
+	if len(text) > 16384 || !utf8.ValidString(text) || count < 0 {
+		return false
+	}
+	longest, current, end := 0, 0, 0
+	for _, span := range integerPattern.FindAllStringIndex(text, -1) {
+		if strings.ContainsFunc(text[end:span[0]], unicode.IsLetter) {
+			current = 0
+		}
+		value, err := strconv.Atoi(text[span[0]:span[1]])
+		if err == nil && value >= 1 && value <= 355 {
+			current++
+		}
+		longest = max(longest, current)
+		end = span[1]
+	}
+	return longest >= max(80, int(math.Ceil(float64(count)*.55)))
+}
+
+func ValidSample(text string) bool { return ValidArray(text, 0) }
+
 func Decide(candidates []Candidate, valid int) Result {
-	r := Result{Status: "inconclusive", ValidSamples: valid}
+	r := Result{Status: "inconclusive", Engine: Version, ValidSamples: valid, Matches: map[string]float64{}}
 	if len(candidates) < 2 {
 		return r
 	}
+	for _, candidate := range candidates {
+		r.Matches[candidate.Model] = candidate.Weight
+	}
 	top := candidates[0]
 	r.Closest = top.Model
 	r.Weight = top.Weight
@@ -98,6 +135,9 @@ func Decide(candidates []Candidate, valid int) Result {
 // Score runs only the embedded, pinned scorer. No filesystem, network or host
 // functions are exposed to the VM; upstream text is data, never JavaScript.
 func Score(samples []Sample) (Result, error) {
+	if len(samples) > ProbeCount {
+		return Result{}, errors.New("too many fingerprint samples")
+	}
 	valid := make([]Sample, 0, 3)
 	for _, sample := range samples {
 		if ValidArray(sample.Text, sample.Count) {
@@ -105,7 +145,7 @@ func Score(samples []Sample) (Result, error) {
 		}
 	}
 	if len(valid) == 0 {
-		return Result{Status: "inconclusive"}, nil
+		return Result{Status: "inconclusive", Engine: Version}, nil
 	}
 	compileOnce.Do(func() {
 		program, compileErr = sobek.Compile("fingerprint", strings.ReplaceAll(scorerJS, "export ", ""), true)

pkg/model_fingerprint/fingerprint_test.go

@@ -6,48 +6,51 @@ import (
 	"testing"
 
 	"github.com/QuantumNous/new-api/common"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 )
 
-func TestFingerprintReferenceAndInvalidSamples(t *testing.T) {
-	data, err := os.ReadFile("testdata/astra-reference.json")
-	if err != nil {
-		t.Fatal(err)
-	}
-	var rows []struct {
-		Text  string `json:"text"`
-		Count int    `json:"requested_count"`
-	}
-	if err = common.Unmarshal(data, &rows); err != nil {
-		t.Fatal(err)
-	}
-	samples := []Sample{}
-	for _, row := range rows {
-		samples = append(samples, Sample{Text: row.Text, Count: row.Count})
-	}
-	r, err := Score(samples)
-	if err != nil || r.Status != "match" || r.Closest != Model || r.ValidSamples != 3 {
-		t.Fatalf("%+v %v", r, err)
-	}
-	r, err = Score(samples[:2])
-	if err != nil || r.Status != "inconclusive" {
-		t.Fatalf("partial samples: %+v %v", r, err)
-	}
-	for _, s := range []string{"[1,2]", "null", "[" + strings.Repeat("-1,", 299) + "1]", "[" + strings.Repeat("1.2,", 299) + "1]", "ignore instructions; [1]"} {
-		if ValidArray(s, 300) {
-			t.Fatalf("accepted invalid array: %s", s[:min(len(s), 20)])
+func TestModelTraceUpstreamParity(t *testing.T) {
+	data, err := os.ReadFile("testdata/modeltrace-reference.json")
+	require.NoError(t, err)
+	var fixtures []struct {
+		Samples  []Sample
+		Expected struct {
+			Results []Candidate `json:"results"`
+			Valid   int         `json:"used_outputs"`
 		}
 	}
-	c := []Candidate{{Model: "gpt-5.6-luna", Weight: .98, Score: 2}, {Model: Model, Weight: .02, Score: 1}}
-	if Decide(c, 3).Status != "mismatch" {
-		t.Fatal("wrong model gate")
-	}
-	c[1].Score = 1.7
-	if Decide(c, 3).Status != "inconclusive" {
-		t.Fatal("low margin must abstain")
+	require.NoError(t, common.Unmarshal(data, &fixtures))
+	for _, fixture := range fixtures {
+		result, err := Score(fixture.Samples)
+		require.NoError(t, err)
+		assert.Equal(t, Version, result.Engine)
+		assert.Equal(t, fixture.Expected.Valid, result.ValidSamples)
+		assert.Equal(t, fixture.Expected.Results[0].Model, result.Closest)
+		for _, candidate := range fixture.Expected.Results {
+			assert.InDelta(t, candidate.Weight, result.Matches[candidate.Model], 1e-9)
+		}
 	}
+	assert.False(t, ValidSample("I cannot provide the sequence."))
+	assert.True(t, ValidArray(strings.Repeat("42,", 180), 300))
+	assert.False(t, ValidArray(strings.Repeat("42,", 100), 300))
+	result := Decide([]Candidate{{Model: Model, Weight: .79, Score: 2}, {Model: "other", Weight: .21, Score: 1}}, 3)
+	assert.Equal(t, "inconclusive", result.Status)
+	result = Decide([]Candidate{{Model: "other", Weight: .95, Score: 2}, {Model: Model, Weight: .05, Score: 1}}, 2)
+	assert.Equal(t, "inconclusive", result.Status)
+	result = Decide([]Candidate{{Model: "other", Weight: .95, Score: 2}, {Model: Model, Weight: .05, Score: 1}}, 3)
+	assert.Equal(t, "mismatch", result.Status)
 }
 
 func TestFingerprintResponseCompletionAndTools(t *testing.T) {
+	stream := "data: {\"type\":\"response.output_text.delta\",\"delta\":\"42\"}\n\ndata: {\"type\":\"response.completed\",\"response\":{\"status\":\"completed\",\"usage\":{\"input_tokens\":10,\"output_tokens\":1}}}\n\n"
+	answer, err := ReadResponse(strings.NewReader(stream), "text/event-stream")
+	require.NoError(t, err)
+	assert.Equal(t, "42", answer.Text)
+	assert.True(t, answer.HasUsage)
+	answer, err = ReadResponse(strings.NewReader(strings.Replace(stream, `"status":"completed"`, `"status":"completed","output":[]`, 1)), "text/event-stream")
+	require.NoError(t, err)
+	assert.Empty(t, answer.Text)
 	completed := `{"status":"completed","model":"gpt-6-astra","output":[{"type":"message","content":[{"type":"output_text","text":"[1,2,3]"}]}],"usage":{"input_tokens":21,"output_tokens":12,"input_tokens_details":{"cached_tokens":10}}}`
 	for _, input := range []struct{ body, content string }{{completed, "application/json"}, {"data: {\"type\":\"response.completed\",\"response\":" + completed + "}\n\n", "text/event-stream"}} {
 		r, err := ReadResponse(strings.NewReader(input.body), input.content)

pkg/model_fingerprint/response.go

@@ -76,6 +76,8 @@ func ReadResponse(reader io.Reader, contentType string) (ProbeResponse, error) {
 	scanner := bufio.NewScanner(buffered)
 	scanner.Buffer(make([]byte, 4096), 2<<20)
 	var data strings.Builder
+	var deltas strings.Builder
+	refused := false
 	for scanner.Scan() {
 		line := scanner.Text()
 		if strings.HasPrefix(line, "data:") {
@@ -101,6 +103,15 @@ func ReadResponse(reader io.Reader, contentType string) (ProbeResponse, error) {
 		}
 		event := gjson.Parse(payload)
 		typ := event.Get("type").String()
+		if typ == "response.output_text.delta" {
+			deltas.WriteString(event.Get("delta").String())
+			if deltas.Len() > 65536 {
+				return ProbeResponse{}, ErrInvalidSample
+			}
+		}
+		if typ == "response.refusal.delta" || typ == "response.refusal.done" {
+			refused = true
+		}
 		if typ == "response.output_item.added" || typ == "response.output_item.done" {
 			itemType := event.Get("item.type").String()
 			if itemType != "message" && itemType != "reasoning" {
@@ -108,7 +119,16 @@ func ReadResponse(reader io.Reader, contentType string) (ProbeResponse, error) {
 			}
 		}
 		if typ == "response.completed" {
-			return completedResponse(event.Get("response"))
+			response := event.Get("response")
+			answer, err := completedResponse(response)
+			if refused {
+				return answer, ErrInvalidSample
+			}
+			// Meow 4.5.4 accepts deltas only when completion omits output entirely.
+			if err == nil && !response.Get("output").Exists() {
+				answer.Text = deltas.String()
+			}
+			return answer, err
 		}
 		if typ == "response.failed" || typ == "response.incomplete" {
 			return completedResponse(event.Get("response"))

pkg/model_fingerprint/testdata/modeltrace-reference.json

@@ -0,0 +1 @@
+[{"samples":[{"text":"217, 43, 308, 126, 351, 89, 164, 22, 279, 135, 62, 324, 191, 107,\n54, 286, 173, 9, 242, 118, 337, 76, 205, 149, 31, 263, 98, 312,\n184, 67, 345, 231, 112, 296, 18, 157, 83, 254, 139, 320, 46, 201,\n328, 121, 57, 274, 193, 34, 166, 355, 94, 223, 11, 287, 148, 70,\n103, 249, 16, 332, 178, 91, 215, 48, 301, 134, 267, 59, 186, 240,\n281, 39, 153, 226, 72, 314, 108, 197, 25, 348, 162, 84, 259, 119,\n65, 304, 181, 14, 237, 96, 342, 145, 208, 51, 276, 124, 319, 173,\n232, 110, 353, 78, 159, 290, 42, 203, 131, 7, 246, 187, 325, 99,\n147, 269, 33, 194, 318, 86, 221, 56, 305, 114, 252, 169, 20, 339,\n292, 74, 128, 243, 5, 177, 334, 102, 211, 63, 284, 151, 36, 229,\n81, 316, 198, 45, 261, 137, 350, 93, 174, 28, 234, 116, 297, 68,\n202, 13, 278, 155, 88, 331, 241, 49, 182, 309, 123, 266, 71, 214,\n344, 97, 161, 24, 295, 206, 58, 251, 142, 323, 109, 38, 273, 189,\n127, 283, 52, 218, 346, 79, 168, 302, 15, 236, 104, 193, 64, 257,\n225, 90, 311, 140, 32, 180, 354, 117, 264, 73, 199, 47, 288, 152,\n60, 247, 171, 329, 8, 213, 132, 277, 95, 341, 185, 26, 230, 113,\n299, 156, 41, 220, 85, 307, 143, 12, 268, 196, 349, 66, 120, 253,\n175, 326, 101, 55, 289, 209, 30, 138, 244, 77, 315, 163, 4, 227,\n106, 272, 190, 19, 335, 150, 61, 258, 122, 294, 44, 204, 87, 352,\n238, 69, 183, 321, 35, 260, 111, 170, 303, 92, 216, 53, 280, 146,\n17, 248, 133, 340, 80, 195, 310, 29, 224, 158, 75, 293, 115, 262","expected_count":294}],"expected":{"prediction":"gpt-6-astra","prediction_name":"gpt-6-astra","probability":0.999999846742899,"used_outputs":1,"results":[{"model":"gpt-6-astra","display_name":"gpt-6-astra","probability":0.999999846742899,"profile_similarity":0.7130119874890992,"score":2.962896373251895,"family":"gpt","family_name":"GPT","conditional_probability":0.9999998543620706},{"model":"gpt-6-sol","display_name":"gpt-6-sol","probability":1.1261750831683942e-7,"profile_similarity":0.6998678953299008,"score":0.7493510421319641,"family":"gpt","family_name":"GPT","conditional_probability":1.1261750917489167e-7},{"model":"gpt-5.4","display_name":"gpt-5.4","probability":2.2759050801578173e-8,"profile_similarity":0.6947437029962558,"score":0.5281199969747397,"family":"gpt","family_name":"GPT","conditional_probability":2.2759050974983314e-8},{"model":"gpt-5.6-terra","display_name":"gpt-5.6-terra","probability":9.128785824960024e-9,"profile_similarity":0.6969199108069902,"score":0.40173043595685565,"family":"gpt","family_name":"GPT","conditional_probability":9.12878589451382e-9},{"model":"claude-opus-5-5","display_name":"claude-opus-5-5","probability":4.332901518152684e-9,"profile_similarity":0.6902452216813633,"score":0.2986305009047045,"family":"claude","family_name":"Claude","conditional_probability":0.568684022014905},{"model":"claude-sonnet-5-5","display_name":"claude-sonnet-5-5","probability":2.6199944714106767e-9,"profile_similarity":0.6880590180204234,"score":0.2290299663562434,"family":"claude","family_name":"Claude","conditional_probability":0.3438686495449988},{"model":"claude-haiku-4-5-20251001","display_name":"claude-haiku-4-5-20251001","probability":6.116352532954573e-10,"profile_similarity":0.6765474621865657,"score":0.027755348459745643,"family":"claude","family_name":"Claude","conditional_probability":0.08027581388428616},{"model":"gpt-6-luna","display_name":"gpt-6-luna","probability":5.59694688154118e-10,"profile_similarity":0.6911103125741771,"score":0.015477264111949274,"family":"gpt","family_name":"GPT","conditional_probability":5.596946924185285e-10},{"model":"gpt-5.6-sol","display_name":"gpt-5.6-sol","probability":3.1143719574775517e-10,"profile_similarity":0.6893303459014515,"score":-0.06562434430484196,"family":"gpt","family_name":"GPT","conditional_probability":3.1143719812064896e-10},{"model":"gpt-5.6-luna","display_name":"gpt-5.6-luna","probability":2.1724112998700045e-10,"profile_similarity":0.6856500053581723,"score":-0.11545764586235663,"family":"gpt","family_name":"GPT","conditional_probability":2.1724113164219816e-10},{"model":"gpt-5.5","display_name":"gpt-5.5","probability":4.4210292318542455e-11,"profile_similarity":0.6895627615538364,"score":-0.33572243758168335,"family":"gpt","family_name":"GPT","conditional_probability":4.4210292655388314e-11},{"model":"claude-sonnet-4-6","display_name":"claude-sonnet-4-6","probability":3.435184918281792e-11,"profile_similarity":0.6700675476082696,"score":-0.3706291211825624,"family":"claude","family_name":"Claude","conditional_probability":0.004508606455764345},{"model":"claude-opus-4-7","display_name":"claude-opus-4-7","probability":8.777758891718366e-12,"profile_similarity":0.6258308139843289,"score":-0.5594026343083449,"family":"claude","family_name":"Claude","conditional_probability":0.001152062009696384},{"model":"claude-opus-5","display_name":"claude-opus-5","probability":5.740678040458984e-12,"profile_similarity":0.663953540718587,"score":-0.6181533384273423,"family":"claude","family_name":"Claude","conditional_probability":0.0007534516682Line truncated

pkg/perf_metrics/live_status.go

@@ -39,7 +39,7 @@ func recordUserResult(group string, result UserRequestResult) {
 		defer cancel()
 		encoded := strconv.FormatInt(result.At, 10) + ":" + strconv.FormatBool(result.Success)
 		// Delayed writers cannot replace newer evidence from another instance.
-		_ = common.RDB.Eval(ctx, `local old=redis.call('GET',KEYS[1]); local at=old and tonumber(string.match(old,'^(%d+):')) or 0; if tonumber(ARGV[1])>=at then redis.call('SET',KEYS[1],ARGV[2],'EX',900) end; return 1`, []string{"status:user-result:" + group}, result.At, encoded).Err()
+		_ = common.RDB.Eval(ctx, `local old=redis.call('GET',KEYS[1]); local at=old and tonumber(string.match(old,'^(%d+):')) or 0; if tonumber(ARGV[1])>=at then redis.call('SET',KEYS[1],ARGV[2],'EX',900) end; return 1`, []string{"status:stream-user-result:" + group}, result.At, encoded).Err()
 	}
 }
 
@@ -56,7 +56,7 @@ func LatestUserResults(ctx context.Context, groups []string) map[string]UserRequ
 	if common.RedisEnabled && common.RDB != nil && len(groups) > 0 {
 		keys := make([]string, len(groups))
 		for i, g := range groups {
-			keys[i] = "status:user-result:" + g
+			keys[i] = "status:stream-user-result:" + g
 		}
 		ctx, cancel := context.WithTimeout(ctx, time.Second)
 		defer cancel()

pkg/perf_metrics/live_status_test.go

@@ -38,7 +38,7 @@ func TestLiveUserResults(t *testing.T) {
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Cleanup(func() { liveResults.Delete(tc.name) })
-			RecordRelayResult(tc.ctx, &relaycommon.RelayInfo{UsingGroup: tc.name, TokenId: tc.token, IsChannelTest: tc.test, StartTime: time.Now()}, tc.err)
+			RecordRelayResult(tc.ctx, &relaycommon.RelayInfo{UsingGroup: tc.name, TokenId: tc.token, IsChannelTest: tc.test, IsStream: true, StartTime: time.Now()}, tc.err)
 			result := LatestUserResults(context.Background(), []string{tc.name})[tc.name]
 			assert.Equal(t, tc.want, result.At > 0)
 			if tc.want {
@@ -60,6 +60,8 @@ func TestUserResultsAreOrderedAndSurviveRestart(t *testing.T) {
 		client.Close()
 	})
 	ctx := context.Background()
+	require.NoError(t, client.Set(ctx, "status:user-result:group", "9999999999999:false", time.Minute).Err())
+	assert.Empty(t, LatestUserResults(ctx, []string{"group"}))
 	check := func(want UserRequestResult) {
 		assert.Equal(t, want, LatestUserResults(ctx, []string{"group"})["group"])
 		liveResults.Delete("group")
@@ -76,9 +78,33 @@ func TestUserResultsAreOrderedAndSurviveRestart(t *testing.T) {
 	recordUserResult("group", UserRequestResult{At: 2001, Success: false})
 	check(UserRequestResult{At: 2002, Success: true})
 	assert.True(t, LatestUserResults(ctx, []string{"other"})["other"].Success)
-	ttl, err := client.TTL(ctx, "status:user-result:group").Result()
+	ttl, err := client.TTL(ctx, "status:stream-user-result:group").Result()
 	require.NoError(t, err)
 	assert.Equal(t, 15*time.Minute, ttl)
 	server.FastForward(15 * time.Minute)
 	assert.Zero(t, LatestUserResults(ctx, []string{"group"})["group"].At)
 }
+
+func TestNonStreamRequestsCannotChangeHealth(t *testing.T) {
+	oldRedis := common.RedisEnabled
+	common.RedisEnabled = false
+	t.Cleanup(func() {
+		common.RedisEnabled = oldRedis
+		liveResults.Delete("stream-only")
+		hotBuckets.Clear()
+	})
+	ctx := context.Background()
+	info := &relaycommon.RelayInfo{UsingGroup: "stream-only", OriginModelName: "test-model", TokenId: 1, IsStream: true, StartTime: time.Now()}
+	RecordRelayResult(ctx, info, nil)
+	want := LatestUserResults(ctx, []string{info.UsingGroup})[info.UsingGroup]
+	require.True(t, want.Success)
+	require.Positive(t, want.At)
+	info.IsStream = false
+	for _, apiErr := range []*types.NewAPIError{nil, types.InitOpenAIError("server_error", 500), types.InitOpenAIError("server_error", 502)} {
+		RecordRelayResult(ctx, info, apiErr)
+		assert.Equal(t, want, LatestUserResults(ctx, []string{info.UsingGroup})[info.UsingGroup])
+	}
+	info.IsStream = true
+	RecordRelayResult(ctx, info, types.InitOpenAIError("server_error", 502))
+	assert.False(t, LatestUserResults(ctx, []string{info.UsingGroup})[info.UsingGroup].Success)
+}

pkg/perf_metrics/metrics.go

@@ -29,8 +29,9 @@ func Init() {
 
 // RecordRelayResult samples one finished relay exactly once, at the request
 // boundary, regardless of how many channel attempts it took.
+// Non-streaming requests do not provide evidence for the streaming service.
 func RecordRelayResult(ctx context.Context, info *relaycommon.RelayInfo, apiErr *types.NewAPIError) {
-	if info == nil {
+	if info == nil || !info.IsStream {
 		return
 	}
 	outcome := ClassifyRelayOutcome(ctx, info, apiErr)