269 lines
9.8 KiB
Go
269 lines
9.8 KiB
Go
package controller
|
|
|
|
import (
|
|
"context"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/QuantumNous/new-api/common"
|
|
"github.com/QuantumNous/new-api/model"
|
|
perfmetrics "github.com/QuantumNous/new-api/pkg/perf_metrics"
|
|
"github.com/QuantumNous/new-api/service"
|
|
"github.com/QuantumNous/new-api/setting/ratio_setting"
|
|
|
|
"github.com/gin-gonic/gin"
|
|
)
|
|
|
|
const requestTimeout = 30 * time.Second
|
|
|
|
type Monitor struct {
|
|
VisionUnavailable bool `json:"vision_unavailable,omitempty"`
|
|
QualityResetAt int64 `json:"quality_reset_at,omitempty"`
|
|
Tags []string `json:"tags,omitempty"`
|
|
RecentDegraded bool `json:"recent_degraded,omitempty"`
|
|
ModelCosts map[string]model.GroupModelCost `json:"model_costs,omitempty"`
|
|
ModelPerformance map[string]model.GroupModelPerformance `json:"model_performance,omitempty"`
|
|
GroupRatio float64 `json:"group_ratio"`
|
|
ProbeHistory []service.ProbeHistoryPoint `json:"probe_history"`
|
|
ID string `json:"id"`
|
|
Models []string `json:"models"`
|
|
Alias string `json:"alias,omitempty"`
|
|
Hidden bool `json:"hidden,omitempty"`
|
|
Ciallo *service.BenchmarkSummary `json:"ciallo,omitempty"`
|
|
Vision *service.BenchmarkSummary `json:"vision,omitempty"`
|
|
Benchmark *service.BenchmarkSummary `json:"benchmark,omitempty"`
|
|
StatusSource string `json:"status_source"`
|
|
StatusCheckedAt int64 `json:"status_checked_at,omitempty"`
|
|
Fingerprint *service.AstraSamplingSummary `json:"fingerprint,omitempty"`
|
|
Metrics *perfmetrics.GroupResult `json:"metrics,omitempty"`
|
|
Name string `json:"name"`
|
|
Uptime float64 `json:"uptime"`
|
|
Status int `json:"status"`
|
|
Group string `json:"group,omitempty"`
|
|
Ping int `json:"ping,omitempty"`
|
|
LastChecked string `json:"last_checked,omitempty"`
|
|
History []MonitorSample `json:"history,omitempty"`
|
|
CacheStats *model.GroupCacheStats `json:"cache_stats,omitempty"`
|
|
ProbeStatus int `json:"probe_status"`
|
|
ChannelDisabled bool `json:"channel_disabled"`
|
|
}
|
|
|
|
type MonitorSample struct {
|
|
Status int `json:"status"`
|
|
Ping int `json:"ping,omitempty"`
|
|
Time string `json:"time"`
|
|
}
|
|
|
|
type UptimeGroupResult struct {
|
|
CategoryName string `json:"categoryName"`
|
|
Monitors []Monitor `json:"monitors"`
|
|
}
|
|
|
|
var groupCacheSnapshot struct {
|
|
sync.Mutex
|
|
updated time.Time
|
|
channels string
|
|
stats map[string]model.GroupCacheStats
|
|
metrics map[string]perfmetrics.GroupResult
|
|
}
|
|
|
|
const requestStatusFreshness = 2 * time.Hour
|
|
|
|
func currentGroupMonitors(ctx context.Context, admin bool) ([]UptimeGroupResult, error) {
|
|
channels, err := model.GetStatusChannels(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
settings, err := model.GetStatusPageGroups(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
models, err := model.GetStatusPageModels(ctx, channels)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
usable := service.GetUserUsableGroups("")
|
|
now := time.Now()
|
|
fingerprints, fingerprintErr := service.AstraSamplingSummaries(ctx, now, settings)
|
|
if fingerprintErr != nil {
|
|
return nil, fingerprintErr
|
|
}
|
|
detections, detectionErr := service.GroupDetectionSummaries(ctx, now, fingerprints, settings)
|
|
if detectionErr != nil {
|
|
return nil, detectionErr
|
|
}
|
|
active := make(map[string]bool)
|
|
order := make([]string, 0)
|
|
for _, channel := range channels {
|
|
for _, group := range strings.Split(channel.Group, ",") {
|
|
group = strings.TrimSpace(group)
|
|
if _, visible := usable[group]; (!visible || settings[group].Hidden) && !admin {
|
|
continue
|
|
}
|
|
if _, exists := active[group]; !exists {
|
|
order = append(order, group)
|
|
}
|
|
active[group] = active[group] || channel.Status == common.ChannelStatusEnabled
|
|
}
|
|
}
|
|
encoded, err := common.Marshal(channels)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
groupCacheSnapshot.Lock()
|
|
defer groupCacheSnapshot.Unlock()
|
|
if time.Since(groupCacheSnapshot.updated) >= time.Minute || groupCacheSnapshot.channels != string(encoded) {
|
|
now := time.Now()
|
|
stats, statsErr := model.GetGroupCacheStats(ctx, channels, now.Add(-24*time.Hour), now)
|
|
if statsErr != nil {
|
|
return nil, statsErr
|
|
}
|
|
groupCacheSnapshot.stats = stats
|
|
names := []string{}
|
|
if err := model.DB.WithContext(ctx).Model(&model.Ability{}).Where("enabled = ?", true).Distinct().Pluck("model", &names).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
performance, metricsErr := perfmetrics.QueryGroups(names, 24)
|
|
if metricsErr != nil {
|
|
return nil, metricsErr
|
|
}
|
|
groupCacheSnapshot.metrics = make(map[string]perfmetrics.GroupResult)
|
|
for _, group := range performance.Groups {
|
|
groupCacheSnapshot.metrics[group.Group] = group
|
|
}
|
|
groupCacheSnapshot.updated = now
|
|
groupCacheSnapshot.channels = string(encoded)
|
|
}
|
|
visionUnavailable, qualityErr := service.VisionUnavailableGroups(ctx)
|
|
if qualityErr != nil {
|
|
return nil, qualityErr
|
|
}
|
|
userResults := perfmetrics.LatestUserResults(ctx, order)
|
|
monitors := make([]Monitor, 0, len(order))
|
|
for _, group := range order {
|
|
monitor := Monitor{QualityResetAt: settings[group].ResetBefore, ID: model.DetectionSecretHash(group)[:16], Name: group, Status: -1, Models: models[group], GroupRatio: ratio_setting.GetGroupRatio(group)}
|
|
if monitor.Models == nil {
|
|
monitor.Models = []string{}
|
|
}
|
|
monitor.ModelPerformance = make(map[string]model.GroupModelPerformance, len(monitor.Models))
|
|
for _, modelName := range monitor.Models {
|
|
performance, performanceErr := service.GroupModelPerformance(ctx, group, modelName)
|
|
if performanceErr != nil {
|
|
return nil, performanceErr
|
|
}
|
|
monitor.ModelPerformance[modelName] = performance
|
|
}
|
|
if settings[group].Alias != "" {
|
|
monitor.Name = settings[group].Alias
|
|
}
|
|
if admin {
|
|
monitor.Alias, monitor.Hidden = settings[group].Alias, settings[group].Hidden
|
|
}
|
|
if admin {
|
|
monitor.Group = group
|
|
}
|
|
monitor.Fingerprint = fingerprints[group]
|
|
if monitor.Fingerprint == nil {
|
|
monitor.Fingerprint = &service.AstraSamplingSummary{Enabled: service.AstraSamplingEnabled(), Available: fingerprintErr == nil, History: []service.AstraSamplingHistory{}}
|
|
}
|
|
monitor.ProbeStatus = monitor.Status
|
|
monitor.ChannelDisabled = !active[group]
|
|
if !active[group] {
|
|
monitor.Status = 0
|
|
}
|
|
stats := groupCacheSnapshot.stats[group]
|
|
monitor.CacheStats = &stats
|
|
applyRequestMetrics(&monitor, groupCacheSnapshot.metrics[group])
|
|
if !admin && monitor.Metrics != nil {
|
|
monitor.Metrics.Group = monitor.Name
|
|
}
|
|
monitor.Benchmark = nil
|
|
monitor.Vision = detections[group].Vision
|
|
monitor.VisionUnavailable = visionUnavailable[group]
|
|
monitor.Ciallo = detections[group].Ciallo
|
|
monitor.ProbeHistory = detections[group].ProbeHistory
|
|
applyCurrentStatus(&monitor, userResults[group], now.Unix())
|
|
monitors = append(monitors, monitor)
|
|
}
|
|
return []UptimeGroupResult{{CategoryName: "Groups", Monitors: monitors}}, nil
|
|
}
|
|
|
|
func GetGroupStatus(c *gin.Context) {
|
|
ctx, cancel := context.WithTimeout(c.Request.Context(), requestTimeout)
|
|
defer cancel()
|
|
|
|
results, err := currentGroupMonitors(ctx, false)
|
|
if err != nil {
|
|
common.SysError("group status statistics: " + err.Error())
|
|
c.JSON(http.StatusServiceUnavailable, gin.H{"success": false, "message": "Status data is temporarily unavailable"})
|
|
return
|
|
}
|
|
c.JSON(http.StatusOK, gin.H{"success": true, "message": "", "data": results})
|
|
}
|
|
|
|
func requestStatus(rate float64) int {
|
|
if rate >= 90 {
|
|
return 1
|
|
}
|
|
if rate >= 70 {
|
|
return 2
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func applyCurrentStatus(monitor *Monitor, result perfmetrics.UserRequestResult, now int64) {
|
|
monitor.Status, monitor.StatusCheckedAt = -1, 0
|
|
monitor.StatusSource = "detection"
|
|
if monitor.ChannelDisabled {
|
|
monitor.Status = 0
|
|
monitor.StatusSource = "disabled"
|
|
return
|
|
}
|
|
// Current availability uses dedicated probes, not quality tests or hourly
|
|
// request ratios. Allow one missed 15-minute probe before evidence expires.
|
|
for _, probe := range monitor.ProbeHistory {
|
|
if probe.At > 0 && probe.At >= now-1800 && probe.At <= now && probe.At >= monitor.StatusCheckedAt {
|
|
monitor.Status = probe.Status
|
|
monitor.StatusCheckedAt = probe.At
|
|
}
|
|
}
|
|
// The latest real user result is stronger evidence than synthetic checks
|
|
// on another member channel, whether it succeeded or failed.
|
|
if at := result.At / 1000; at > 0 && at >= now-900 && at <= now {
|
|
monitor.Status = 0
|
|
if result.Success {
|
|
monitor.Status = 1
|
|
}
|
|
monitor.StatusSource = "requests"
|
|
monitor.StatusCheckedAt = at
|
|
}
|
|
}
|
|
|
|
func applyRequestMetrics(monitor *Monitor, metrics perfmetrics.GroupResult) {
|
|
monitor.Status = -1
|
|
monitor.Uptime = 0
|
|
monitor.Ping = 0
|
|
monitor.LastChecked = ""
|
|
monitor.History = nil
|
|
if len(metrics.Series) > 0 {
|
|
monitor.Metrics = &metrics
|
|
monitor.Uptime = metrics.SuccessRate / 100
|
|
latest := metrics.Series[len(metrics.Series)-1]
|
|
if time.Since(time.Unix(latest.Ts, 0)) <= requestStatusFreshness {
|
|
monitor.Status = requestStatus(latest.SuccessRate)
|
|
}
|
|
for _, point := range metrics.Series {
|
|
monitor.History = append(monitor.History, MonitorSample{
|
|
Status: requestStatus(point.SuccessRate), Ping: int(point.AvgTtftMs),
|
|
Time: time.Unix(point.Ts, 0).UTC().Format("2006-01-02 15:04:05"),
|
|
})
|
|
}
|
|
monitor.LastChecked = monitor.History[len(monitor.History)-1].Time
|
|
}
|
|
if monitor.ChannelDisabled {
|
|
monitor.Status = 0
|
|
}
|
|
}
|