feat: sync deployed v1.0.0-rc.38-custom.42

Sync production source including routing, billing, Astra detection and public uptime. Update custom build instructions and license packaging.

service/relay_error.go

@@ -0,0 +1,122 @@
+package service
+
+import (
+	"fmt"
+	"time"
+
+	"github.com/QuantumNous/new-api/common"
+	"github.com/QuantumNous/new-api/constant"
+	"github.com/QuantumNous/new-api/logger"
+	"github.com/QuantumNous/new-api/model"
+	relaycommon "github.com/QuantumNous/new-api/relay/common"
+	"github.com/QuantumNous/new-api/relaykit/types"
+	"github.com/QuantumNous/new-api/setting/operation_setting"
+
+	"github.com/bytedance/gopkg/util/gopool"
+	"github.com/gin-gonic/gin"
+)
+
+// DecideRelayRetry is the single retry decision for relay attempts. The reason
+// is recorded in the request policy decision events of the log details.
+func DecideRelayRetry(c *gin.Context, err *types.NewAPIError, retryTimes int) PolicyDecision {
+	if err == nil {
+		return PolicyDecision{Action: "stop", Reason: "request_completed", Source: "system"}
+	}
+	if ShouldSkipRetryAfterChannelAffinityFailure(c) {
+		source := RequestPolicy(c).SessionModeSource
+		if source == "" {
+			source = "session_rule"
+		}
+		return PolicyDecision{Action: "stop", Reason: "strict_session", Source: source}
+	}
+	if GetChannelConstraints(c).SuppressesRetry() {
+		return PolicyDecision{Action: "stop", Reason: "pinned_channel", Source: "channel_constraint"}
+	}
+	if types.IsChannelError(err) {
+		return PolicyDecision{Action: "retry", Reason: "channel_error", Source: "system"}
+	}
+	if types.IsSkipRetryError(err) {
+		return PolicyDecision{Action: "stop", Reason: "non_retryable_error", Source: "system"}
+	}
+	if retryTimes <= 0 {
+		return PolicyDecision{Action: "stop", Reason: "attempt_budget_exhausted", Source: "global"}
+	}
+	code := err.StatusCode
+	if code >= 200 && code < 300 {
+		return PolicyDecision{Action: "stop", Reason: "system_retry_exclusion", Source: "system"}
+	}
+	if code < 100 || code > 599 {
+		return PolicyDecision{Action: "retry", Reason: "unrecognized_status", Source: "system"}
+	}
+	if operation_setting.IsAlwaysSkipRetryCode(err.GetErrorCode()) || operation_setting.IsAlwaysSkipRetryStatusCode(code) {
+		return PolicyDecision{Action: "stop", Reason: "system_retry_exclusion", Source: "system"}
+	}
+	if operation_setting.ShouldRetryByStatusCode(code) {
+		return PolicyDecision{Action: "retry", Reason: "retry_status_matched", Source: "global"}
+	}
+	return PolicyDecision{Action: "stop", Reason: "status_not_retryable", Source: "global"}
+}
+
+func ShouldRetryRelayError(c *gin.Context, openaiErr *types.NewAPIError, retryTimes int) bool {
+	return DecideRelayRetry(c, openaiErr, retryTimes).Action == "retry"
+}
+
+func ProcessChannelError(c *gin.Context, channelError types.ChannelError, err *types.NewAPIError, relayInfo *relaycommon.RelayInfo) {
+	if err == nil {
+		return
+	}
+	logger.LogError(c, fmt.Sprintf("channel error (channel #%d, status code: %d): %s", channelError.ChannelId, err.StatusCode, common.LocalLogPreview(err.MaskSensitiveErrorWithStatusCode())))
+	if err.GetErrorCode() != relaycommon.ResponseGuardErrorCode && ShouldDisableChannel(err) && channelError.AutoBan {
+		reason := err.MaskSensitiveErrorWithStatusCode()
+		gopool.Go(func() {
+			DisableChannel(channelError, reason)
+		})
+	}
+
+	if constant.ErrorLogEnabled && types.IsRecordErrorLog(err) {
+		userId := c.GetInt("id")
+		tokenName := c.GetString("token_name")
+		modelName := c.GetString("original_model")
+		tokenId := c.GetInt("token_id")
+		usingGroup := relayErrorGroup(c, channelError.ChannelId, relayInfo)
+		other := model.NewLogOther()
+		if c.Request != nil && c.Request.URL != nil {
+			other.SetPublic("request_path", c.Request.URL.Path)
+		}
+		other.SetPublic("error_type", err.GetErrorType())
+		other.SetPublic("error_code", err.GetErrorCode())
+		other.SetPublic("status_code", err.StatusCode)
+		AppendRelayLogAdminInfo(c, relayInfo, other)
+		AppendResponseModelLogInfo(relayInfo, other)
+		AppendTaskPluginContextAuditInfo(c, other)
+		startTime := common.GetContextKeyTime(c, constant.ContextKeyRequestStartTime)
+		if startTime.IsZero() {
+			startTime = time.Now()
+		}
+		useTimeSeconds := int(time.Since(startTime).Seconds())
+		other.SetRoot("upstream_error", err.MaskSensitiveErrorWithStatusCode())
+		content := fmt.Sprintf("status_code=%d, The model service is temporarily unavailable. Please try again later.", err.StatusCode)
+		model.RecordErrorLog(c, userId, channelError.ChannelId, modelName, tokenName, content, tokenId, useTimeSeconds, common.GetContextKeyBool(c, constant.ContextKeyIsStream), usingGroup, other)
+	}
+}
+
+// A user/token's default group is not necessarily the selected retry group.
+func relayErrorGroup(c *gin.Context, channelID int, info *relaycommon.RelayInfo) string {
+	if channelID <= 0 {
+		return ""
+	}
+	if info != nil && info.UsingGroup != "" && info.UsingGroup != "auto" {
+		return info.UsingGroup
+	}
+	if group := common.GetContextKeyString(c, constant.ContextKeyAutoGroup); group != "" {
+		return group
+	}
+	if group := RequestPolicy(c).SelectedGroup; group != "" {
+		return group
+	}
+	group := common.GetContextKeyString(c, constant.ContextKeyUsingGroup)
+	if group == "auto" {
+		return ""
+	}
+	return group
+}