new-api-custom/pkg/model_benchmark/fingerprint.go

// Meow predictive fingerprint integration, pinned to the public GPT benchmark.
// Required Notice: Copyright 2026 chen-006 and contributors.
// Original project: https://github.com/chen-006/gpt56_api_detector
// See LICENSE-meow.txt for the PolyForm Noncommercial 1.0.0 license.
package model_benchmark

import (
	_ "embed"
	"errors"
	"math"
	"sort"
	"strings"
	"sync"
	"unicode/utf8"

	"github.com/QuantumNous/new-api/common"
	"golang.org/x/text/cases"
)

const Model = "gpt-6-astra"
const Version = "meow-4.5.4-predictive.20260924.2"
const ProbeCount = 32
const OutputTokens = 128

//go:embed meow.json
var benchmarkJSON []byte

type Sample struct {
	Text string `json:"text"`
	// Ordinal in the fixed quick-tier plan, under the existing storage key.
	Count int `json:"expected_count"`
}

type Result struct {
	Status       string             `json:"status"`
	ValidSamples int                `json:"valid_samples"`
	Closest      string             `json:"closest,omitempty"`
	Weight       float64            `json:"relative_weight,omitempty"`
	Gap          float64            `json:"score_gap,omitempty"`
	Engine       string             `json:"engine,omitempty"`
	Matches      map[string]float64 `json:"matches,omitempty"`
	Thresholds   map[string]float64 `json:"thresholds,omitempty"`
}

type Probe struct {
	ID     string `json:"id"`
	System string `json:"system"`
	Prompt string `json:"prompt"`
}
type fittedCell struct {
	Categories []string             `json:"categories"`
	Alpha      map[string][]float64 `json:"alpha"`
}
type benchmark struct {
	Plan   []Probe `json:"-"`
	Probes []struct {
		Cells []Probe `json:"cells"`
	} `json:"probes"`
	Fitted struct {
		Models     []string              `json:"models"`
		Sources    []string              `json:"sources"`
		References []string              `json:"reference_sources"`
		Cells      map[string]fittedCell `json:"cells"`
	} `json:"fitted"`
	Tiers map[string]struct {
		Counts     map[string]int     `json:"counts"`
		Thresholds map[string]float64 `json:"thresholds"`
	} `json:"tiers"`
}

var load = sync.OnceValues(func() (*benchmark, error) {
	var b benchmark
	if err := common.Unmarshal(benchmarkJSON, &b); err != nil {
		return nil, err
	}
	var plan []Probe
	for _, p := range b.Probes {
		for _, c := range p.Cells {
			for range b.Tiers["low"].Counts[c.ID] {
				plan = append(plan, c)
			}
		}
	}
	sort.SliceStable(plan, func(i, j int) bool { return plan[i].ID < plan[j].ID })
	if len(plan) != ProbeCount {
		return nil, errors.New("invalid meow plan")
	}
	b.Plan = plan
	return &b, nil
})

func ProbeAt(index int) (Probe, error) {
	b, err := load()
	if err != nil {
		return Probe{}, err
	}
	if index < 0 || index >= len(b.Plan) {
		return Probe{}, errors.New("invalid probe index")
	}
	return b.Plan[index], nil
}

func ValidSample(text string) bool {
	if !utf8.ValidString(text) || utf8.RuneCountInString(text) > 65536 {
		return false
	}
	text = cases.Fold().String(strings.TrimSpace(text))
	return text != "" && utf8.RuneCountInString(text) <= 4096
}

// Whole-run Dirichlet posterior prediction, using average log evidence versus
// the strongest rival and calibrated thresholds. Closest alone is no verdict.
func Score(samples []Sample) (Result, error) {
	b, err := load()
	if err != nil {
		return Result{}, err
	}
	r := Result{Status: "inconclusive", Engine: Version, Matches: map[string]float64{}, Thresholds: b.Tiers["low"].Thresholds}
	counts := map[string]map[string]int{}
	seen := map[int]bool{}
	for _, sample := range samples {
		if sample.Count < 0 || sample.Count >= len(b.Plan) || seen[sample.Count] {
			return r, errors.New("invalid or duplicate meow sample")
		}
		seen[sample.Count] = true
		if !ValidSample(sample.Text) {
			continue
		}
		id := b.Plan[sample.Count].ID
		text := cases.Fold().String(strings.TrimSpace(sample.Text))
		if counts[id] == nil {
			counts[id] = map[string]int{}
		}
		counts[id][text]++
		r.ValidSamples++
	}
	sufficient := r.ValidSamples >= int(math.Ceil(.6*ProbeCount))
	evidence := map[string]float64{}
	for id, planned := range b.Tiers["low"].Counts {
		cell := b.Fitted.Cells[id]
		numbers := make([]int, len(cell.Categories))
		positions := map[string]int{}
		for i, category := range cell.Categories {
			positions[category] = i
		}
		n := 0
		for text, number := range counts[id] {
			i, ok := positions[text]
			if !ok {
				i = positions["__UNSEEN_IN_TRAINING__"]
			}
			numbers[i] += number
			n += number
		}
		if n < int(math.Ceil(.6*float64(planned))) {
			sufficient = false
		}
		for _, source := range b.Fitted.Sources {
			alpha := cell.Alpha[source]
			total, value := 0.0, 0.0
			for i, a := range alpha {
				total += a
				left, _ := math.Lgamma(a + float64(numbers[i]))
				right, _ := math.Lgamma(a)
				value += left - right
			}
			left, _ := math.Lgamma(total)
			right, _ := math.Lgamma(total + float64(n))
			evidence[source] += value + left - right
		}
	}
	evidence["other_known_external"] = math.Inf(-1)
	for _, source := range b.Fitted.References {
		evidence["other_known_external"] = math.Max(evidence["other_known_external"], evidence[source])
	}
	for _, model := range b.Fitted.Models {
		rival := math.Inf(-1)
		for _, other := range b.Fitted.Models {
			if other != model {
				rival = math.Max(rival, evidence[other])
			}
		}
		r.Matches[model] = 1 / (1 + math.Exp((rival-evidence[model])/float64(max(r.ValidSamples, 1))))
	}
	ordered := append([]string(nil), b.Fitted.Models...)
	sort.SliceStable(ordered, func(i, j int) bool { return r.Matches[ordered[i]] > r.Matches[ordered[j]] })
	top := ordered[0]
	r.Weight = r.Matches[top]
	r.Gap = (evidence[top] - evidence[ordered[1]]) / float64(max(r.ValidSamples, 1))
	if r.ValidSamples > 0 {
		r.Closest = top
	}
	if sufficient && r.Weight > r.Thresholds[top] && r.Weight > r.Matches[ordered[1]] {
		r.Status = "mismatch"
		if top == Model {
			r.Status = "match"
		}
	}
	return r, nil
}