cneicy/new-api-custom
1
0 Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

feat: sync deployed v1.0.0-rc.38-custom.67

Browse Source
This commit is contained in:
cneicy committed 2026-10-06 00:07:10 +08:00
1 parent d09216bbde
commit 8e6ed0317b
154 files changed
+39540 -20509
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

service/token_routing.go
+77 -6
View File
@@ -1,6 +1,7 @@
package service
import (
"crypto/sha256"
"fmt"
"strconv"
"strings"
@@ -17,8 +18,9 @@ import (
)
type poolBinding struct {
Channel int `json:"channel"`
Group string `json:"group"`
PolicyFingerprint string `json:"policy_fingerprint"`
Channel int `json:"channel"`
Group string `json:"group"`
}
var poolBindingOnce sync.Once
@@ -44,6 +46,18 @@ func PreferredPoolChannel(c *gin.Context, modelName string) (*model.Channel, str
key := common.GenerateHMAC(fmt.Sprintf("%d:%d:%d:%s:%s", c.GetInt("id"), c.GetInt("token_id"), len(modelName), modelName, meta.CacheKey))
c.Set("pool_affinity_key", key)
c.Set("pool_affinity_ttl", meta.TTLSeconds)
// Snapshot the routing policy before selection. An older in-flight request
// may finish after a manual switch, but its binding must not override the
// new policy on the next request.
policyJSON, err := common.Marshal(struct {
Policy *model.TokenRoutingPolicy
Pool []model.TokenGroupEntry
}{RequestRoutingPolicy(c), ResolveRequestPool(c, modelName)})
if err != nil {
return nil, ""
}
policyFingerprint := fmt.Sprintf("%x", sha256.Sum256(policyJSON))
c.Set("pool_affinity_policy_fingerprint", policyFingerprint)
binding, found, err := poolBindings().Get(key)
if err != nil {
return nil, ""
@@ -61,6 +75,11 @@ func PreferredPoolChannel(c *gin.Context, modelName string) (*model.Channel, str
return nil, ""
}
var other struct {
Admin struct {
Details struct {
PolicyFingerprint string `json:"routing_policy_fingerprint"`
} `json:"request_details"`
} `json:"admin_info"`
Result string `json:"model_guard_result"`
Stream struct {
Status string `json:"status"`
@@ -75,15 +94,33 @@ func PreferredPoolChannel(c *gin.Context, modelName string) (*model.Channel, str
if previous.IsStream && (other.Stream.Status != "ok" || (other.Stream.EndReason != "done" && other.Stream.EndReason != "eof") || (other.Stream.Response != "" && other.Stream.Response != "completed")) {
return nil, ""
}
binding = poolBinding{Channel: previous.ChannelId, Group: previous.Group}
binding = poolBinding{Channel: previous.ChannelId, Group: previous.Group, PolicyFingerprint: other.Admin.Details.PolicyFingerprint}
}
// Unversioned legacy bindings cannot prove they belong to this policy.
// This check also prevents historical logs from restoring an old route.
if binding.PolicyFingerprint != policyFingerprint {
ClearPoolAffinity(c)
return nil, ""
}
for _, entry := range ResolveRequestPool(c, modelName) {
if entry.Group != binding.Group {
continue
}
if requestGroupDegraded(c, modelName, entry.Group) {
ClearPoolAffinity(c)
recordDirectSelection(c, nil, binding.Channel, entry.Group, modelName, "pool_group_degraded")
return nil, ""
}
if poolGroupSkipped(c, modelName, entry.Group) || poolHigherPriorityRecovered(c, modelName, entry.Group) {
ClearPoolAffinity(c)
return nil, ""
}
channel, err := model.CacheGetChannel(binding.Channel)
if err == nil && channel.Status == common.ChannelStatusEnabled && model.IsChannelEnabledForGroupModel(entry.Group, modelName, channel.Id) {
if ok, _ := model.ChannelSatisfiesFilters(channel, modelName, GetChannelConstraints(c).Filters); ok {
if !claimPoolGroup(c, modelName, entry.Group) {
return nil, ""
}
common.SetContextKey(c, constant.ContextKeyAutoGroup, entry.Group)
MarkChannelAffinityUsed(c, entry.Group, channel.Id)
recordDirectSelection(c, channel, channel.Id, entry.Group, modelName, "pool_affinity_hit")
@@ -112,6 +149,7 @@ func RecordPoolAffinity(c *gin.Context) {
if !RequestPolicy(c).Successful {
return
}
RecordPoolSuccess(c, c.GetString("original_model"), common.GetContextKeyString(c, constant.ContextKeyAutoGroup))
key := c.GetString("pool_affinity_key")
if key == "" {
return
@@ -120,7 +158,7 @@ func RecordPoolAffinity(c *gin.Context) {
if ttl <= 0 {
ttl = 3600
}
_ = poolBindings().SetWithTTL(key, poolBinding{Channel: c.GetInt("channel_id"), Group: common.GetContextKeyString(c, constant.ContextKeyAutoGroup)}, time.Duration(ttl)*time.Second)
_ = poolBindings().SetWithTTL(key, poolBinding{Channel: c.GetInt("channel_id"), Group: common.GetContextKeyString(c, constant.ContextKeyAutoGroup), PolicyFingerprint: c.GetString("pool_affinity_policy_fingerprint")}, time.Duration(ttl)*time.Second)
}
func RequestRoutingPolicy(c *gin.Context) *model.TokenRoutingPolicy {
@@ -135,11 +173,14 @@ func ResolveRequestPool(c *gin.Context, modelName string) []model.TokenGroupEntr
return nil
}
userGroup := common.GetContextKeyString(c, constant.ContextKeyUserGroup)
if policy.Mode == "smart" {
return WithImageGenerationGroup(c, modelName, resolveAutomaticPool(c, modelName))
}
inherited := GetUserAutoGroup(userGroup)
if policy.Mode == "user" {
inherited = []string{userGroup}
}
return policy.Resolve(modelName, inherited, func(group string) bool { return IsUserSelectableGroup(userGroup, group) })
return WithImageGenerationGroup(c, modelName, policy.Resolve(modelName, inherited, func(group string) bool { return IsUserSelectableGroup(userGroup, group) }))
}
func selectTokenRoutingPool(param *RetryParam) (*model.Channel, string, error) {
@@ -152,6 +193,25 @@ func selectTokenRoutingPool(param *RetryParam) (*model.Channel, string, error) {
excluded = append(excluded, id)
}
filters = append(filters, dto.ChannelFilter{Kind: dto.FilterAttemptedChannel, ExcludedChannelIDs: excluded})
// Retry once in the current group, including single-channel groups. Preserve
// every security/protocol filter; remove only the request's attempted list.
if retryGroup := param.Ctx.GetString("pool_retry_group"); retryGroup != "" && poolFailures(param.Ctx)[retryGroup] == 1 && !poolGroupSkipped(param.Ctx, param.ModelName, retryGroup) && !requestGroupDegraded(param.Ctx, param.ModelName, retryGroup) {
param.Ctx.Set("pool_retry_group", "")
for _, entry := range entries {
if entry.Group != retryGroup {
continue
}
channel, err := model.GetRandomSatisfiedChannel(retryGroup, param.ModelName, 0, constraints.Filters)
if err != nil {
return nil, retryGroup, err
}
if channel != nil {
common.SetContextKey(param.Ctx, constant.ContextKeyAutoGroup, retryGroup)
recordDirectSelection(param.Ctx, channel, channel.Id, retryGroup, param.ModelName, "pool_same_group_retry")
return channel, retryGroup, nil
}
}
}
for start := 0; start < len(entries); {
end := start + 1
for end < len(entries) && entries[end].Priority == entries[start].Priority {
@@ -165,7 +225,12 @@ func selectTokenRoutingPool(param *RetryParam) (*model.Channel, string, error) {
offset := nextTierIndex(fmt.Sprintf("pool:%d:%s:%s", param.Ctx.GetInt("token_id"), param.ModelName, strings.Join(names, ",")), len(groups))
for i := range len(groups) {
group := groups[(offset+i)%len(groups)].Group
if !IsGroupAvailable(group) {
if poolGroupSkipped(param.Ctx, param.ModelName, group) {
recordDirectSelection(param.Ctx, nil, 0, group, param.ModelName, "pool_group_cooldown")
continue
}
if requestGroupDegraded(param.Ctx, param.ModelName, group) {
recordDirectSelection(param.Ctx, nil, 0, group, param.ModelName, "pool_group_degraded")
continue
}
trace := &model.SelectionTrace{}
@@ -174,6 +239,9 @@ func selectTokenRoutingPool(param *RetryParam) (*model.Channel, string, error) {
return nil, group, err
}
if channel != nil {
if !claimPoolGroup(param.Ctx, param.ModelName, group) {
continue
}
common.SetContextKey(param.Ctx, constant.ContextKeyAutoGroup, group)
recordDirectSelection(param.Ctx, channel, channel.Id, group, param.ModelName, "model_pool_priority")
return channel, group, nil
@@ -181,5 +249,8 @@ func selectTokenRoutingPool(param *RetryParam) (*model.Channel, string, error) {
}
start = end
}
if param.Ctx.GetBool("pool_degradation_unavailable") {
return nil, param.TokenGroup, fmt.Errorf("group quality temporarily unavailable")
}
return nil, param.TokenGroup, nil
}