fix: follow live request results and prioritize usable groups

controller/live_status_test.go

@@ -1,28 +1,65 @@
 package controller
 
 import (
+	perfmetrics "github.com/QuantumNous/new-api/pkg/perf_metrics"
+	"github.com/QuantumNous/new-api/service"
 	"github.com/stretchr/testify/assert"
 	"testing"
-	"time"
 )
 
-func TestSuccessfulUserRequestRestoresStatus(t *testing.T) {
-	now := time.Now().Unix()
+func TestCurrentGroupStatus(t *testing.T) {
+	const now int64 = 10000
 	for _, tc := range []struct {
-		name           string
-		success, probe int64
-		disabled       bool
-		want           int
+		name     string
+		success  int64
+		probes   []service.ProbeHistoryPoint
+		disabled bool
+		want     int
+		source   string
+		at       int64
 	}{
-		{"fresh user overrides failed probe", now, now - 10, false, 1},
-		{"newer failed probe wins", now - 10, now, false, 0},
-		{"expired success", now - 901, now - 1000, false, 0},
-		{"disabled remains disabled", now, now - 10, true, 0},
+		{"fresh user overrides older failed probe", now, []service.ProbeHistoryPoint{{At: now - 10, Status: 0}}, false, 1, "requests", now},
+		{"newer failed probe cannot erase recent user success", now - 10, []service.ProbeHistoryPoint{{At: now, Status: 0}}, false, 1, "requests", now - 10},
+		{"working primary with failing backup stays normal", now - 60, []service.ProbeHistoryPoint{{At: now, Status: 2}}, false, 1, "requests", now - 60},
+		{"success remains fresh for full fifteen minutes", now - 900, nil, false, 1, "requests", now - 900},
+		{"expired user success uses latest partial probe", now - 901, []service.ProbeHistoryPoint{{At: now - 10, Status: 2}}, false, 2, "detection", now - 10},
+		{"healthy probe overrides historical hourly error", 0, []service.ProbeHistoryPoint{{At: now - 100, Status: 1}}, false, 1, "detection", now - 100},
+		{"latest probe determines availability", 0, []service.ProbeHistoryPoint{{At: now - 100, Status: 0}, {At: now - 200, Status: 1}}, false, 0, "detection", now - 100},
+		{"one missed probe remains fresh", 0, []service.ProbeHistoryPoint{{At: now - 1800, Status: 1}}, false, 1, "detection", now - 1800},
+		{"stale probe is unknown", 0, []service.ProbeHistoryPoint{{At: now - 1801, Status: 0}}, false, -1, "detection", 0},
+		{"quality failures alone do not imply an outage", 0, nil, false, -1, "detection", 0},
+		{"disabled remains disabled", now, []service.ProbeHistoryPoint{{At: now, Status: 1}}, true, 0, "disabled", 0},
+		{"future evidence is ignored", now + 1, []service.ProbeHistoryPoint{{At: now + 1, Status: 1}}, false, -1, "detection", 0},
 	} {
 		t.Run(tc.name, func(t *testing.T) {
-			m := Monitor{Status: 0, ChannelDisabled: tc.disabled}
-			applyUserSuccess(&m, tc.success, tc.probe, now)
+			m := Monitor{Status: 0, StatusSource: "requests", StatusCheckedAt: now - 100, ChannelDisabled: tc.disabled, ProbeHistory: tc.probes, Uptime: 0.58, Fingerprint: &service.AstraSamplingSummary{Errors: 25}, Pelican: &service.PelicanSummary{Failed: 33}}
+			applyCurrentStatus(&m, perfmetrics.UserRequestResult{At: tc.success * 1000, Success: true}, now)
 			assert.Equal(t, tc.want, m.Status)
+			assert.Equal(t, tc.source, m.StatusSource)
+			assert.Equal(t, tc.at, m.StatusCheckedAt)
+			assert.Equal(t, 25, m.Fingerprint.Errors)
+			assert.Equal(t, 33, m.Pelican.Failed)
+			assert.Equal(t, 0.58, m.Uptime)
+		})
+	}
+}
+
+func TestFailedUserRequestChangesGroupStatus(t *testing.T) {
+	const now int64 = 10000
+	for _, tc := range []struct {
+		name   string
+		at     int64
+		want   int
+		source string
+	}{
+		{"user failure overrides a newer healthy probe", now - 10, 0, "requests"},
+		{"user failure expires after fifteen minutes", now - 901, 1, "detection"},
+	} {
+		t.Run(tc.name, func(t *testing.T) {
+			m := Monitor{Status: 1, ProbeHistory: []service.ProbeHistoryPoint{{At: now, Status: 1}}}
+			applyCurrentStatus(&m, perfmetrics.UserRequestResult{At: tc.at * 1000, Success: false}, now)
+			assert.Equal(t, tc.want, m.Status)
+			assert.Equal(t, tc.source, m.StatusSource)
 		})
 	}
 }

controller/uptime_kuma.go

@@ -126,7 +126,7 @@ func currentGroupMonitors(ctx context.Context, admin bool) ([]UptimeGroupResult,
 		groupCacheSnapshot.updated = now
 		groupCacheSnapshot.channels = string(encoded)
 	}
-	successes := perfmetrics.LatestUserSuccess(ctx, order)
+	userResults := perfmetrics.LatestUserResults(ctx, order)
 	monitors := make([]Monitor, 0, len(order))
 	for _, group := range order {
 		monitor := Monitor{ID: model.DetectionSecretHash(group)[:16], Name: group, Status: -1, Models: models[group]}
@@ -158,16 +158,8 @@ func currentGroupMonitors(ctx context.Context, admin bool) ([]UptimeGroupResult,
 			monitor.Metrics.Group = monitor.Name
 		}
 		monitor.Pelican = detections[group].Pelican
-		monitor.StatusSource = "requests"
-		if monitor.ChannelDisabled {
-			monitor.StatusSource = "disabled"
-		} else if monitor.Status < 0 {
-			monitor.Status = detections[group].Status
-			monitor.StatusSource = "detection"
-			monitor.StatusCheckedAt = detections[group].At
-		}
 		monitor.ProbeHistory = detections[group].ProbeHistory
-		applyUserSuccess(&monitor, successes[group], detections[group].At, now.Unix())
+		applyCurrentStatus(&monitor, userResults[group], now.Unix())
 		monitors = append(monitors, monitor)
 	}
 	return []UptimeGroupResult{{CategoryName: "Groups", Monitors: monitors}}, nil
@@ -196,11 +188,31 @@ func requestStatus(rate float64) int {
 	return 0
 }
 
-func applyUserSuccess(monitor *Monitor, success, probe, now int64) {
-	if !monitor.ChannelDisabled && success > 0 && success >= now-900 && success >= probe {
-		monitor.Status = 1
+func applyCurrentStatus(monitor *Monitor, result perfmetrics.UserRequestResult, now int64) {
+	monitor.Status, monitor.StatusCheckedAt = -1, 0
+	monitor.StatusSource = "detection"
+	if monitor.ChannelDisabled {
+		monitor.Status = 0
+		monitor.StatusSource = "disabled"
+		return
+	}
+	// Current availability uses dedicated probes, not quality tests or hourly
+	// request ratios. Allow one missed 15-minute probe before evidence expires.
+	for _, probe := range monitor.ProbeHistory {
+		if probe.At > 0 && probe.At >= now-1800 && probe.At <= now && probe.At >= monitor.StatusCheckedAt {
+			monitor.Status = probe.Status
+			monitor.StatusCheckedAt = probe.At
+		}
+	}
+	// The latest real user result is stronger evidence than synthetic checks
+	// on another member channel, whether it succeeded or failed.
+	if at := result.At / 1000; at > 0 && at >= now-900 && at <= now {
+		monitor.Status = 0
+		if result.Success {
+			monitor.Status = 1
+		}
 		monitor.StatusSource = "requests"
-		monitor.StatusCheckedAt = success
+		monitor.StatusCheckedAt = at
 	}
 }