61 lines
1.9 KiB
Go
61 lines
1.9 KiB
Go
package service
|
|
|
|
import (
|
|
"sync"
|
|
"time"
|
|
|
|
fingerprint "github.com/QuantumNous/new-api/pkg/model_fingerprint"
|
|
"github.com/gin-gonic/gin"
|
|
)
|
|
|
|
var groupDegradationSnapshot struct {
|
|
sync.Mutex
|
|
updated time.Time
|
|
latest map[string]AstraSamplingHistory
|
|
vision map[string]DetectionHistory
|
|
ciallo map[string]DetectionHistory
|
|
}
|
|
|
|
func requestGroupQuality(c *gin.Context) (map[string]AstraSamplingHistory, map[string]DetectionHistory, map[string]DetectionHistory) {
|
|
if c.GetBool("pool_degradation_unavailable") {
|
|
return nil, nil, nil
|
|
}
|
|
latest, found := c.Get("pool_group_fingerprints")
|
|
vision, _ := c.Get("pool_group_visions")
|
|
ciallo, _ := c.Get("pool_group_ciallos")
|
|
if !found {
|
|
rounds, qualities, ciallos, err := cachedGroupQuality(c.Request.Context())
|
|
if err != nil {
|
|
c.Set("pool_degradation_unavailable", true)
|
|
return nil, nil, nil
|
|
}
|
|
latest, vision, ciallo = rounds, qualities, ciallos
|
|
c.Set("pool_group_fingerprints", latest)
|
|
c.Set("pool_group_visions", vision)
|
|
c.Set("pool_group_ciallos", ciallo)
|
|
}
|
|
rounds, _ := latest.(map[string]AstraSamplingHistory)
|
|
visions, _ := vision.(map[string]DetectionHistory)
|
|
ciallos, _ := ciallo.(map[string]DetectionHistory)
|
|
return rounds, visions, ciallos
|
|
}
|
|
|
|
func requestGroupDegraded(c *gin.Context, modelName, group string) bool {
|
|
policy := RequestRoutingPolicy(c)
|
|
if policy == nil {
|
|
return false
|
|
}
|
|
skip := policy.SkipDegradedGroups && (modelName == fingerprint.Model || policy.SkipDegradedNonAstra)
|
|
if policy.Mode == "smart" {
|
|
skip = !policy.AllowDegraded && (modelName == fingerprint.Model || policy.SkipDegradedNonAstra)
|
|
}
|
|
if !skip {
|
|
return false
|
|
}
|
|
rounds, visions, ciallos := requestGroupQuality(c)
|
|
h, ok := rounds[group]
|
|
return c.GetBool("pool_degradation_unavailable") ||
|
|
(ok && h.ValidSamples > 0 && h.Closest != "" && h.Closest != fingerprint.Model) ||
|
|
visions[group].Quality == "degraded" || ciallos[group].Quality == "degraded"
|
|
}
|