feat: sync deployed v1.0.0-rc.38-custom.67
This commit is contained in:
cneicy committed
1 parent
d09216bbde
commit
8e6ed0317b
154 files changed
+39540
-20509
No files matched your search
@@ -0,0 +1,200 @@
|
||||
// Meow predictive fingerprint integration, pinned to the public GPT benchmark.
|
||||
// Required Notice: Copyright 2026 chen-006 and contributors.
|
||||
// Original project: https://github.com/chen-006/gpt56_api_detector
|
||||
// See LICENSE-meow.txt for the PolyForm Noncommercial 1.0.0 license.
|
||||
package model_benchmark
|
||||
|
||||
import (
|
||||
_ "embed"
|
||||
"errors"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"golang.org/x/text/cases"
|
||||
)
|
||||
|
||||
const Model = "gpt-6-astra"
|
||||
const Version = "meow-4.5.4-predictive.20260924.2"
|
||||
const ProbeCount = 32
|
||||
const OutputTokens = 128
|
||||
|
||||
//go:embed meow.json
|
||||
var benchmarkJSON []byte
|
||||
|
||||
type Sample struct {
|
||||
Text string `json:"text"`
|
||||
// Ordinal in the fixed quick-tier plan, under the existing storage key.
|
||||
Count int `json:"expected_count"`
|
||||
}
|
||||
|
||||
type Result struct {
|
||||
Status string `json:"status"`
|
||||
ValidSamples int `json:"valid_samples"`
|
||||
Closest string `json:"closest,omitempty"`
|
||||
Weight float64 `json:"relative_weight,omitempty"`
|
||||
Gap float64 `json:"score_gap,omitempty"`
|
||||
Engine string `json:"engine,omitempty"`
|
||||
Matches map[string]float64 `json:"matches,omitempty"`
|
||||
Thresholds map[string]float64 `json:"thresholds,omitempty"`
|
||||
}
|
||||
|
||||
type Probe struct {
|
||||
ID string `json:"id"`
|
||||
System string `json:"system"`
|
||||
Prompt string `json:"prompt"`
|
||||
}
|
||||
type fittedCell struct {
|
||||
Categories []string `json:"categories"`
|
||||
Alpha map[string][]float64 `json:"alpha"`
|
||||
}
|
||||
type benchmark struct {
|
||||
Plan []Probe `json:"-"`
|
||||
Probes []struct {
|
||||
Cells []Probe `json:"cells"`
|
||||
} `json:"probes"`
|
||||
Fitted struct {
|
||||
Models []string `json:"models"`
|
||||
Sources []string `json:"sources"`
|
||||
References []string `json:"reference_sources"`
|
||||
Cells map[string]fittedCell `json:"cells"`
|
||||
} `json:"fitted"`
|
||||
Tiers map[string]struct {
|
||||
Counts map[string]int `json:"counts"`
|
||||
Thresholds map[string]float64 `json:"thresholds"`
|
||||
} `json:"tiers"`
|
||||
}
|
||||
|
||||
var load = sync.OnceValues(func() (*benchmark, error) {
|
||||
var b benchmark
|
||||
if err := common.Unmarshal(benchmarkJSON, &b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var plan []Probe
|
||||
for _, p := range b.Probes {
|
||||
for _, c := range p.Cells {
|
||||
for range b.Tiers["low"].Counts[c.ID] {
|
||||
plan = append(plan, c)
|
||||
}
|
||||
}
|
||||
}
|
||||
sort.SliceStable(plan, func(i, j int) bool { return plan[i].ID < plan[j].ID })
|
||||
if len(plan) != ProbeCount {
|
||||
return nil, errors.New("invalid meow plan")
|
||||
}
|
||||
b.Plan = plan
|
||||
return &b, nil
|
||||
})
|
||||
|
||||
func ProbeAt(index int) (Probe, error) {
|
||||
b, err := load()
|
||||
if err != nil {
|
||||
return Probe{}, err
|
||||
}
|
||||
if index < 0 || index >= len(b.Plan) {
|
||||
return Probe{}, errors.New("invalid probe index")
|
||||
}
|
||||
return b.Plan[index], nil
|
||||
}
|
||||
|
||||
func ValidSample(text string) bool {
|
||||
if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 65536 {
|
||||
return false
|
||||
}
|
||||
text = cases.Fold().String(strings.TrimSpace(text))
|
||||
return text != "" && utf8.RuneCountInString(text) <= 4096
|
||||
}
|
||||
|
||||
// Whole-run Dirichlet posterior prediction, using average log evidence versus
|
||||
// the strongest rival and calibrated thresholds. Closest alone is no verdict.
|
||||
func Score(samples []Sample) (Result, error) {
|
||||
b, err := load()
|
||||
if err != nil {
|
||||
return Result{}, err
|
||||
}
|
||||
r := Result{Status: "inconclusive", Engine: Version, Matches: map[string]float64{}, Thresholds: b.Tiers["low"].Thresholds}
|
||||
counts := map[string]map[string]int{}
|
||||
seen := map[int]bool{}
|
||||
for _, sample := range samples {
|
||||
if sample.Count < 0 || sample.Count >= len(b.Plan) || seen[sample.Count] {
|
||||
return r, errors.New("invalid or duplicate meow sample")
|
||||
}
|
||||
seen[sample.Count] = true
|
||||
if !ValidSample(sample.Text) {
|
||||
continue
|
||||
}
|
||||
id := b.Plan[sample.Count].ID
|
||||
text := cases.Fold().String(strings.TrimSpace(sample.Text))
|
||||
if counts[id] == nil {
|
||||
counts[id] = map[string]int{}
|
||||
}
|
||||
counts[id][text]++
|
||||
r.ValidSamples++
|
||||
}
|
||||
sufficient := r.ValidSamples >= int(math.Ceil(.6*ProbeCount))
|
||||
evidence := map[string]float64{}
|
||||
for id, planned := range b.Tiers["low"].Counts {
|
||||
cell := b.Fitted.Cells[id]
|
||||
numbers := make([]int, len(cell.Categories))
|
||||
positions := map[string]int{}
|
||||
for i, category := range cell.Categories {
|
||||
positions[category] = i
|
||||
}
|
||||
n := 0
|
||||
for text, number := range counts[id] {
|
||||
i, ok := positions[text]
|
||||
if !ok {
|
||||
i = positions["__UNSEEN_IN_TRAINING__"]
|
||||
}
|
||||
numbers[i] += number
|
||||
n += number
|
||||
}
|
||||
if n < int(math.Ceil(.6*float64(planned))) {
|
||||
sufficient = false
|
||||
}
|
||||
for _, source := range b.Fitted.Sources {
|
||||
alpha := cell.Alpha[source]
|
||||
total, value := 0.0, 0.0
|
||||
for i, a := range alpha {
|
||||
total += a
|
||||
left, _ := math.Lgamma(a + float64(numbers[i]))
|
||||
right, _ := math.Lgamma(a)
|
||||
value += left - right
|
||||
}
|
||||
left, _ := math.Lgamma(total)
|
||||
right, _ := math.Lgamma(total + float64(n))
|
||||
evidence[source] += value + left - right
|
||||
}
|
||||
}
|
||||
evidence["other_known_external"] = math.Inf(-1)
|
||||
for _, source := range b.Fitted.References {
|
||||
evidence["other_known_external"] = math.Max(evidence["other_known_external"], evidence[source])
|
||||
}
|
||||
for _, model := range b.Fitted.Models {
|
||||
rival := math.Inf(-1)
|
||||
for _, other := range b.Fitted.Models {
|
||||
if other != model {
|
||||
rival = math.Max(rival, evidence[other])
|
||||
}
|
||||
}
|
||||
r.Matches[model] = 1 / (1 + math.Exp((rival-evidence[model])/float64(max(r.ValidSamples, 1))))
|
||||
}
|
||||
ordered := append([]string(nil), b.Fitted.Models...)
|
||||
sort.SliceStable(ordered, func(i, j int) bool { return r.Matches[ordered[i]] > r.Matches[ordered[j]] })
|
||||
top := ordered[0]
|
||||
r.Weight = r.Matches[top]
|
||||
r.Gap = (evidence[top] - evidence[ordered[1]]) / float64(max(r.ValidSamples, 1))
|
||||
if r.ValidSamples > 0 {
|
||||
r.Closest = top
|
||||
}
|
||||
if sufficient && r.Weight > r.Thresholds[top] && r.Weight > r.Matches[ordered[1]] {
|
||||
r.Status = "mismatch"
|
||||
if top == Model {
|
||||
r.Status = "match"
|
||||
}
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
Reference in new issue
Block a user