feat: sync deployed v1.0.0-rc.38-custom.67

controller/relay.go

@@ -194,9 +194,12 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) {
 	relayInfo.RetryIndex = 0
 	relayInfo.LastError = nil
 	maxRetries := common.RetryTimes
+	poolBudget := service.PoolRetryBudget(c, relayInfo.OriginModelName)
+	poolFailures := 0
+	maxRetries = max(maxRetries, poolBudget)
 	ordinaryRetries := 0
 	if relayInfo.DurableAttemptBilling {
-		maxRetries = 2
+		maxRetries = max(maxRetries, 2)
 		timeout := constant.StreamingTimeout
 		if timeout <= 0 {
 			timeout = 300
@@ -210,7 +213,7 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) {
 			newAPIError = types.NewErrorWithStatusCode(errors.New("response model confirmation budget exhausted"), relaycommon.ResponseGuardErrorCode, http.StatusGatewayTimeout, types.ErrOptionWithSkipRetry())
 			break
 		}
-		if relayInfo.ResponseGuard != nil && relayInfo.UpstreamRequests >= relaycommon.ResponseGuardMaxAttempts {
+		if relayInfo.ResponseGuard != nil && relayInfo.UpstreamRequests-poolFailures >= relaycommon.ResponseGuardMaxAttempts {
 			newAPIError = types.NewErrorWithStatusCode(errors.New("response model attempt budget exhausted"), relaycommon.ResponseGuardErrorCode, http.StatusBadGateway, types.ErrOptionWithSkipRetry())
 			break
 		}
@@ -259,6 +262,9 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) {
 
 		if newAPIError == nil {
 			service.MarkRequestPolicySuccess(c, relayInfo.StreamStatus)
+			if service.RequestPolicy(c).Successful {
+				service.RecordPoolSuccess(c, relayInfo.OriginModelName, currentRequestGroup(c, channel))
+			}
 			relayInfo.LastError = nil
 			service.RecordGroupOutcome(currentRequestGroup(c, channel), true)
 			return
@@ -276,7 +282,16 @@ func Relay(c *gin.Context, relayFormat types.RelayFormat) {
 		if relayInfo.DurableAttemptBilling {
 			remaining = common.RetryTimes - ordinaryRetries
 		}
+		poolFailure := service.IsPoolFailoverError(c, newAPIError)
+		if poolFailure {
+			remaining = maxRetries - retryParam.GetRetry()
+			poolFailures++
+			service.RecordPoolRetryableFailure(c, relayInfo.OriginModelName, currentRequestGroup(c, channel))
+		}
 		decision := service.DecideRelayRetry(c, newAPIError, remaining)
+		if poolFailure && service.PoolRetryExhausted(c, relayInfo.OriginModelName, currentRequestGroup(c, channel)) {
+			decision = service.PolicyDecision{Action: "stop", Reason: "pool_no_backup", Source: "routing"}
+		}
 		if canSwitchRejectedAstra(c, relayInfo, newAPIError) {
 			decision = service.PolicyDecision{Action: "retry", Reason: "model_mismatch_switch", Source: "model_guard"}
 			service.ClearPoolAffinity(c)