feat: sync deployed v1.0.0-rc.38-custom.67

pkg/perf_metrics/live_status.go

@@ -39,7 +39,7 @@ func recordUserResult(group string, result UserRequestResult) {
 		defer cancel()
 		encoded := strconv.FormatInt(result.At, 10) + ":" + strconv.FormatBool(result.Success)
 		// Delayed writers cannot replace newer evidence from another instance.
-		_ = common.RDB.Eval(ctx, `local old=redis.call('GET',KEYS[1]); local at=old and tonumber(string.match(old,'^(%d+):')) or 0; if tonumber(ARGV[1])>=at then redis.call('SET',KEYS[1],ARGV[2],'EX',900) end; return 1`, []string{"status:user-result:" + group}, result.At, encoded).Err()
+		_ = common.RDB.Eval(ctx, `local old=redis.call('GET',KEYS[1]); local at=old and tonumber(string.match(old,'^(%d+):')) or 0; if tonumber(ARGV[1])>=at then redis.call('SET',KEYS[1],ARGV[2],'EX',900) end; return 1`, []string{"status:stream-user-result:" + group}, result.At, encoded).Err()
 	}
 }
 
@@ -56,7 +56,7 @@ func LatestUserResults(ctx context.Context, groups []string) map[string]UserRequ
 	if common.RedisEnabled && common.RDB != nil && len(groups) > 0 {
 		keys := make([]string, len(groups))
 		for i, g := range groups {
-			keys[i] = "status:user-result:" + g
+			keys[i] = "status:stream-user-result:" + g
 		}
 		ctx, cancel := context.WithTimeout(ctx, time.Second)
 		defer cancel()

pkg/perf_metrics/live_status_test.go

@@ -38,7 +38,7 @@ func TestLiveUserResults(t *testing.T) {
 	} {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Cleanup(func() { liveResults.Delete(tc.name) })
-			RecordRelayResult(tc.ctx, &relaycommon.RelayInfo{UsingGroup: tc.name, TokenId: tc.token, IsChannelTest: tc.test, StartTime: time.Now()}, tc.err)
+			RecordRelayResult(tc.ctx, &relaycommon.RelayInfo{UsingGroup: tc.name, TokenId: tc.token, IsChannelTest: tc.test, IsStream: true, StartTime: time.Now()}, tc.err)
 			result := LatestUserResults(context.Background(), []string{tc.name})[tc.name]
 			assert.Equal(t, tc.want, result.At > 0)
 			if tc.want {
@@ -60,6 +60,8 @@ func TestUserResultsAreOrderedAndSurviveRestart(t *testing.T) {
 		client.Close()
 	})
 	ctx := context.Background()
+	require.NoError(t, client.Set(ctx, "status:user-result:group", "9999999999999:false", time.Minute).Err())
+	assert.Empty(t, LatestUserResults(ctx, []string{"group"}))
 	check := func(want UserRequestResult) {
 		assert.Equal(t, want, LatestUserResults(ctx, []string{"group"})["group"])
 		liveResults.Delete("group")
@@ -76,9 +78,33 @@ func TestUserResultsAreOrderedAndSurviveRestart(t *testing.T) {
 	recordUserResult("group", UserRequestResult{At: 2001, Success: false})
 	check(UserRequestResult{At: 2002, Success: true})
 	assert.True(t, LatestUserResults(ctx, []string{"other"})["other"].Success)
-	ttl, err := client.TTL(ctx, "status:user-result:group").Result()
+	ttl, err := client.TTL(ctx, "status:stream-user-result:group").Result()
 	require.NoError(t, err)
 	assert.Equal(t, 15*time.Minute, ttl)
 	server.FastForward(15 * time.Minute)
 	assert.Zero(t, LatestUserResults(ctx, []string{"group"})["group"].At)
 }
+
+func TestNonStreamRequestsCannotChangeHealth(t *testing.T) {
+	oldRedis := common.RedisEnabled
+	common.RedisEnabled = false
+	t.Cleanup(func() {
+		common.RedisEnabled = oldRedis
+		liveResults.Delete("stream-only")
+		hotBuckets.Clear()
+	})
+	ctx := context.Background()
+	info := &relaycommon.RelayInfo{UsingGroup: "stream-only", OriginModelName: "test-model", TokenId: 1, IsStream: true, StartTime: time.Now()}
+	RecordRelayResult(ctx, info, nil)
+	want := LatestUserResults(ctx, []string{info.UsingGroup})[info.UsingGroup]
+	require.True(t, want.Success)
+	require.Positive(t, want.At)
+	info.IsStream = false
+	for _, apiErr := range []*types.NewAPIError{nil, types.InitOpenAIError("server_error", 500), types.InitOpenAIError("server_error", 502)} {
+		RecordRelayResult(ctx, info, apiErr)
+		assert.Equal(t, want, LatestUserResults(ctx, []string{info.UsingGroup})[info.UsingGroup])
+	}
+	info.IsStream = true
+	RecordRelayResult(ctx, info, types.InitOpenAIError("server_error", 502))
+	assert.False(t, LatestUserResults(ctx, []string{info.UsingGroup})[info.UsingGroup].Success)
+}

pkg/perf_metrics/metrics.go

@@ -29,8 +29,9 @@ func Init() {
 
 // RecordRelayResult samples one finished relay exactly once, at the request
 // boundary, regardless of how many channel attempts it took.
+// Non-streaming requests do not provide evidence for the streaming service.
 func RecordRelayResult(ctx context.Context, info *relaycommon.RelayInfo, apiErr *types.NewAPIError) {
-	if info == nil {
+	if info == nil || !info.IsStream {
 		return
 	}
 	outcome := ClassifyRelayOutcome(ctx, info, apiErr)