30 lines
901 B
Go
30 lines
901 B
Go
package service
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/QuantumNous/new-api/model"
|
|
"github.com/samber/hot"
|
|
)
|
|
|
|
var groupModelPerformanceCache = hot.NewHotCache[string, model.GroupModelPerformance](hot.LRU, 2000).WithTTL(time.Minute).Build()
|
|
var groupModelPerformanceMu sync.Mutex
|
|
|
|
// Share one snapshot between status display and automatic throughput ranking.
|
|
func GroupModelPerformance(ctx context.Context, group, modelName string) (model.GroupModelPerformance, error) {
|
|
key := fmt.Sprintf("%p/%p/%q/%q", model.DB, model.LOG_DB, group, modelName)
|
|
groupModelPerformanceMu.Lock()
|
|
defer groupModelPerformanceMu.Unlock()
|
|
if value, found, err := groupModelPerformanceCache.Get(key); err == nil && found {
|
|
return value, nil
|
|
}
|
|
value, err := model.GetGroupModelPerformance(ctx, group, modelName)
|
|
if err == nil {
|
|
groupModelPerformanceCache.Set(key, value)
|
|
}
|
|
return value, err
|
|
}
|