49 lines
1.3 KiB
Go
49 lines
1.3 KiB
Go
package service
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
fingerprint "github.com/QuantumNous/new-api/pkg/model_fingerprint"
|
|
"github.com/gin-gonic/gin"
|
|
)
|
|
|
|
var groupDegradationSnapshot struct {
|
|
sync.Mutex
|
|
updated time.Time
|
|
latest map[string]AstraSamplingHistory
|
|
benchmark map[string]DetectionHistory
|
|
}
|
|
|
|
func requestGroupDegraded(c *gin.Context, modelName, group string) bool {
|
|
policy := RequestRoutingPolicy(c)
|
|
if policy == nil {
|
|
return false
|
|
}
|
|
skip := policy.SkipDegradedGroups && (modelName == fingerprint.Model || policy.SkipDegradedNonAstra)
|
|
if policy.Mode == "smart" {
|
|
skip = !policy.AllowDegraded && (modelName == fingerprint.Model || policy.SkipDegradedNonAstra)
|
|
}
|
|
if !skip {
|
|
return false
|
|
}
|
|
if c.GetBool("pool_degradation_unavailable") {
|
|
return true
|
|
}
|
|
latest, found := c.Get("pool_group_fingerprints")
|
|
benchmark, _ := c.Get("pool_group_benchmarks")
|
|
if !found {
|
|
rounds, qualities, err := cachedGroupQuality(c.Request.Context())
|
|
if err != nil {
|
|
c.Set("pool_degradation_unavailable", true)
|
|
return true
|
|
}
|
|
latest, benchmark = rounds, qualities
|
|
c.Set("pool_group_fingerprints", latest)
|
|
c.Set("pool_group_benchmarks", benchmark)
|
|
}
|
|
rounds, _ := latest.(map[string]AstraSamplingHistory)
|
|
h, ok := rounds[group]
|
|
return (ok && h.ValidSamples > 0 && h.Closest != "" && h.Closest != fingerprint.Model)
|
|
}
|