new-api-custom/pkg/model_fingerprint/fingerprint.go

// Numerical fingerprint scorer adapted from Katao123/gpt-model-check (MIT),
// commit 3d6716a0903cf061daea001d5d44ee410846634e. See the adjacent licenses.
package model_fingerprint

import (
	_ "embed"
	"errors"
	"math"
	"strings"
	"sync"
	"time"

	"github.com/QuantumNous/new-api/common"
	"github.com/grafana/sobek"
)

const Model = "gpt-6-astra"
const Version = "modeltrace-20260905-gmc-3d6716a"

//go:embed bank.json
var bankJSON string

//go:embed scorer.js
var scorerJS string

var compileOnce sync.Once
var program *sobek.Program
var compileErr error

type Sample struct {
	Text  string `json:"text"`
	Count int    `json:"expected_count"`
}

type Candidate struct {
	Model  string  `json:"model"`
	Weight float64 `json:"probability"`
	Score  float64 `json:"score"`
}

type Result struct {
	Status       string  `json:"status"`
	ValidSamples int     `json:"valid_samples"`
	Closest      string  `json:"closest,omitempty"`
	Weight       float64 `json:"relative_weight,omitempty"`
	Gap          float64 `json:"score_gap,omitempty"`
}

func ValidArray(text string, count int) bool {
	if len(text) > 16384 || count <= 0 {
		return false
	}
	text = strings.TrimSpace(text)
	if strings.HasPrefix(text, "```") && strings.HasSuffix(text, "```") {
		if i := strings.IndexByte(text, '\n'); i >= 0 {
			text = strings.TrimSpace(text[i+1 : len(text)-3])
		}
	}
	var numbers []int
	if common.Unmarshal([]byte(text), &numbers) != nil {
		return false
	}
	if len(numbers) < max(80, int(math.Ceil(float64(count)*.55))) || len(numbers) > count*2 {
		return false
	}
	for _, number := range numbers {
		if number < 1 || number > 355 {
			return false
		}
	}
	return true
}

func Decide(candidates []Candidate, valid int) Result {
	r := Result{Status: "inconclusive", ValidSamples: valid}
	if len(candidates) < 2 {
		return r
	}
	top := candidates[0]
	r.Closest = top.Model
	r.Weight = top.Weight
	r.Gap = top.Score - candidates[1].Score
	if valid != 3 || math.IsNaN(r.Gap) || math.IsInf(r.Gap, 0) || top.Weight < .8 || r.Gap < .5 {
		return r
	}
	if top.Model == Model {
		r.Status = "match"
		return r
	}
	for _, candidate := range candidates {
		if candidate.Model == Model && candidate.Weight <= .2 && top.Score-candidate.Score >= .5 {
			r.Status = "mismatch"
		}
	}
	return r
}

// Score runs only the embedded, pinned scorer. No filesystem, network or host
// functions are exposed to the VM; upstream text is data, never JavaScript.
func Score(samples []Sample) (Result, error) {
	valid := make([]Sample, 0, 3)
	for _, sample := range samples {
		if ValidArray(sample.Text, sample.Count) {
			valid = append(valid, sample)
		}
	}
	if len(valid) == 0 {
		return Result{Status: "inconclusive"}, nil
	}
	compileOnce.Do(func() {
		program, compileErr = sobek.Compile("fingerprint", strings.ReplaceAll(scorerJS, "export ", ""), true)
	})
	if compileErr != nil {
		return Result{}, compileErr
	}
	vm := sobek.New()
	timer := time.AfterFunc(3*time.Second, func() { vm.Interrupt("fingerprint scoring timeout") })
	defer timer.Stop()
	if _, err := vm.RunProgram(program); err != nil {
		return Result{}, err
	}
	if err := vm.Set("bankJSON", bankJSON); err != nil {
		return Result{}, err
	}
	encoded, err := common.Marshal(valid)
	if err != nil {
		return Result{}, err
	}
	if err = vm.Set("samplesJSON", string(encoded)); err != nil {
		return Result{}, err
	}
	value, err := vm.RunString("JSON.stringify(analyzeGlobalOutputs(JSON.parse(samplesJSON), JSON.parse(bankJSON)))")
	if err != nil {
		return Result{}, err
	}
	var analysis struct {
		Results []Candidate `json:"results"`
	}
	if err = common.Unmarshal([]byte(value.String()), &analysis); err != nil {
		return Result{}, err
	}
	if len(analysis.Results) < 2 {
		return Result{}, errors.New("missing fingerprint candidates")
	}
	return Decide(analysis.Results, len(valid)), nil
}