feat: publish sanitized custom.97 source

This commit is contained in:
cneicy committed 2026-10-27 20:43:39 +08:00
1 parent 8e6ed0317b
commit 86c0276156
1022 files changed
+69903 -19207

No files matched your search

+6 -7
View File
@@ -2,7 +2,6 @@ package model
import (
"errors"
"fmt"
"sort"
"strings"
"sync"
@@ -97,7 +96,7 @@ func getPriority(group string, model string, retry int) (int, error) {
if len(priorities) == 0 {
// 如果没有查询到优先级,则返回错误
return 0, errors.New("数据库一致性被破坏")
return 0, errors.New("database consistency is broken")
}
// 确定要使用的优先级
@@ -395,7 +394,7 @@ var fixLock = sync.Mutex{}
func FixAbility() (int, int, error) {
lock := fixLock.TryLock()
if !lock {
return 0, 0, errors.New("已经有一个修复任务在运行中,请稍后再试")
return 0, 0, common.NewMessage("A repair task is already running. Please try again later")
}
defer fixLock.Unlock()
@@ -403,13 +402,13 @@ func FixAbility() (int, int, error) {
if common.UsingMainDatabase(common.DatabaseTypeSQLite) {
err := DB.Exec("DELETE FROM abilities").Error
if err != nil {
common.SysLog(fmt.Sprintf("Delete abilities failed: %s", err.Error()))
common.SysLog(common.LogText("Delete abilities failed: %s", err.Error()))
return 0, 0, err
}
} else {
err := DB.Exec("TRUNCATE TABLE abilities").Error
if err != nil {
common.SysLog(fmt.Sprintf("Truncate abilities failed: %s", err.Error()))
common.SysLog(common.LogText("Truncate abilities failed: %s", err.Error()))
return 0, 0, err
}
}
@@ -429,7 +428,7 @@ func FixAbility() (int, int, error) {
// Delete all abilities of this channel
err = DB.Where("channel_id IN ?", ids).Delete(&Ability{}).Error
if err != nil {
common.SysLog(fmt.Sprintf("Delete abilities failed: %s", err.Error()))
common.SysLog(common.LogText("Delete abilities failed: %s", err.Error()))
failCount += len(chunk)
continue
}
@@ -437,7 +436,7 @@ func FixAbility() (int, int, error) {
for _, channel := range chunk {
err = channel.AddAbilities(nil)
if err != nil {
common.SysLog(fmt.Sprintf("Add abilities for channel %d failed: %s", channel.Id, err.Error()))
common.SysLog(common.LogText("Add abilities for channel %d failed: %s", channel.Id, err.Error()))
failCount++
} else {
successCount++
+12 -4
View File
@@ -102,7 +102,7 @@ func RecordAuditLog(c *gin.Context, entry AuditLog) {
switch entry.ActorRole {
case common.RoleCommonUser, common.RoleAdminUser, common.RoleRootUser:
default:
logger.LogError(ctx, fmt.Sprintf("audit actor role unavailable (request_id=%s, actor_role=%d)", entry.RequestId, entry.ActorRole))
logger.LogError(ctx, common.LogText("audit actor role unavailable (request_id=%s, actor_role=%d)", entry.RequestId, entry.ActorRole))
entry.ActorRole = 0 // Unknown actors remain visible to root only.
}
if entry.Username == "" {
@@ -113,14 +113,14 @@ func RecordAuditLog(c *gin.Context, entry AuditLog) {
entry.UserAgent = string(ua[:512])
}
if LOG_DB == nil {
logger.LogError(ctx, fmt.Sprintf("audit log write failed (request_id=%s): log database unavailable", entry.RequestId))
logger.LogError(ctx, common.LogText("audit log write failed (request_id=%s): log database unavailable", entry.RequestId))
return
}
var row any = &entry
if common.UsingLogDatabase(common.DatabaseTypeClickHouse) {
encoded, err := common.Marshal(entry.Other)
if err != nil {
logger.LogError(ctx, fmt.Sprintf("audit log write failed (request_id=%s): %v", entry.RequestId, err))
logger.LogError(ctx, common.LogText("audit log write failed (request_id=%s): %v", entry.RequestId, err))
return
}
// The ClickHouse GORM insert callback passes structs to the native
@@ -132,7 +132,7 @@ func RecordAuditLog(c *gin.Context, entry AuditLog) {
}{AuditLog: entry, EncodedOther: string(encoded)}
}
if err := LOG_DB.Table("audit_logs").Create(row).Error; err != nil {
logger.LogError(ctx, fmt.Sprintf("audit log write failed (request_id=%s): %v", entry.RequestId, err))
logger.LogError(ctx, common.LogText("audit log write failed (request_id=%s): %v", entry.RequestId, err))
}
}
@@ -202,14 +202,21 @@ func GetAuditLogs(filter AuditLogFilter, start, limit, viewerRole int) ([]*Audit
return logs, total, nil
}
// UserAccessTokenStatus describes the legacy users.access_token credential.
//
// Deprecated: 旧版面板访问令牌,仅在升级后的过渡期内使用;删除 users.access_token 列时一并移除。
type UserAccessTokenStatus struct {
Exists bool `json:"exists"`
TokenHint string `json:"token_hint"`
TokenRef string `json:"token_ref"`
CreatedAt *int64 `json:"created_at"`
LastUsedAt *int64 `json:"last_used_at"`
LastUsedIp string `json:"last_used_ip"`
}
// GetUserAccessTokenStatus reads last use of the legacy token from the access audit.
//
// Deprecated: 旧版面板访问令牌,仅在升级后的过渡期内使用;删除 users.access_token 列时一并移除。
func GetUserAccessTokenStatus(userId int) (*UserAccessTokenStatus, error) {
var user User
if err := DB.Select("id", "role", "access_token", "access_token_created_at").First(&user, userId).Error; err != nil {
@@ -220,6 +227,7 @@ func GetUserAccessTokenStatus(userId int) (*UserAccessTokenStatus, error) {
return status, nil
}
status.TokenRef = AccessTokenFingerprint(user.GetAccessToken())
status.TokenHint = AccessTokenHint(user.GetAccessToken())
status.CreatedAt = user.AccessTokenCreatedAt
var latest AuditLog
query := LOG_DB.Select("created_at", "ip").Where("user_id = ? AND token_ref = ? AND category = ?", userId, status.TokenRef, AuditCategoryAccessToken)
+4
View File
@@ -98,7 +98,11 @@ type AuthFlowAuthorization struct {
// ValidateAuthSessionWithTx rechecks the authoritative identity while holding the
// user/session locks until the caller's credential change or flow consumption commits.
// Token-bound identities (pat:<id>) are checked against user_access_tokens instead.
func ValidateAuthSessionWithTx(tx *gorm.DB, identity AuthSessionIdentity) error {
if tokenID, ok := ParseAccessTokenSessionID(identity.SessionID); ok {
return validateAccessTokenIdentityWithTx(tx, tokenID, identity)
}
if identity.UserID <= 0 || identity.SessionID == "" || identity.UserAuthVersion <= 0 || identity.SessionVersion <= 0 {
return ErrUserSessionInactive
}
+20
View File
@@ -0,0 +1,20 @@
package model
// BotBalanceAccount contains only the fields needed by the notification bot.
type BotBalanceAccount struct {
ID int `gorm:"column:id"`
Status int
Quota int
}
func FindBotBalanceAccounts(email string, ids []int) ([]BotBalanceAccount, error) {
accounts := []BotBalanceAccount{}
query := DB.Model(&User{}).Select("id", "status", "quota")
if email != "" {
query = query.Where("LOWER(email) = ?", NormalizeEmail(email)).Limit(2)
} else {
query = query.Where("id IN ?", ids)
}
err := query.Find(&accounts).Error
return accounts, err
}
+24 -19
View File
@@ -105,10 +105,15 @@ func NewChannelSortOptions(sortBy string, sortOrder string, idSort bool) Channel
func (options ChannelSortOptions) Apply(query *gorm.DB) *gorm.DB {
if columnName, ok := channelSortColumns[options.SortBy]; ok {
column := clause.Column{Name: columnName}
if columnName == "priority" {
// Routing treats a missing priority as zero on every database.
column = clause.Column{Name: "COALESCE(priority, 0)", Raw: true}
}
return query.Order(clause.OrderByColumn{
Column: clause.Column{Name: columnName},
Column: column,
Desc: options.SortOrder != "asc",
})
}).Order(clause.OrderByColumn{Column: clause.Column{Name: "id"}, Desc: true})
}
if options.IDSort {
return query.Order(clause.OrderByColumn{
@@ -117,9 +122,9 @@ func (options ChannelSortOptions) Apply(query *gorm.DB) *gorm.DB {
})
}
return query.Order(clause.OrderByColumn{
Column: clause.Column{Name: "priority"},
Column: clause.Column{Name: "COALESCE(priority, 0)", Raw: true},
Desc: true,
})
}).Order(clause.OrderByColumn{Column: clause.Column{Name: "id"}, Desc: true})
}
func resolveChannelSortOptions(idSort bool, sortOptions []ChannelSortOptions) ChannelSortOptions {
@@ -316,7 +321,7 @@ func (channel *Channel) GetOtherInfo() map[string]any {
if channel.OtherInfo != "" {
err := common.Unmarshal([]byte(channel.OtherInfo), &otherInfo)
if err != nil {
common.SysLog(fmt.Sprintf("failed to unmarshal other info: channel_id=%d, tag=%s, name=%s, error=%v", channel.Id, channel.GetTag(), channel.Name, err))
common.SysLog(common.LogText("failed to unmarshal other info: channel_id=%d, tag=%s, name=%s, error=%v", channel.Id, channel.GetTag(), channel.Name, err))
}
}
return otherInfo
@@ -325,7 +330,7 @@ func (channel *Channel) GetOtherInfo() map[string]any {
func (channel *Channel) SetOtherInfo(otherInfo map[string]any) {
otherInfoBytes, err := json.Marshal(otherInfo)
if err != nil {
common.SysLog(fmt.Sprintf("failed to marshal other info: channel_id=%d, tag=%s, name=%s, error=%v", channel.Id, channel.GetTag(), channel.Name, err))
common.SysLog(common.LogText("failed to marshal other info: channel_id=%d, tag=%s, name=%s, error=%v", channel.Id, channel.GetTag(), channel.Name, err))
return
}
channel.OtherInfo = string(otherInfoBytes)
@@ -610,7 +615,7 @@ func (channel *Channel) UpdateResponseTime(responseTime int64) {
ResponseTime: int(responseTime),
}).Error
if err != nil {
common.SysLog(fmt.Sprintf("failed to update response time: channel_id=%d, error=%v", channel.Id, err))
common.SysLog(common.LogText("failed to update response time: channel_id=%d, error=%v", channel.Id, err))
}
}
@@ -620,7 +625,7 @@ func (channel *Channel) UpdateBalance(balance float64) {
Balance: balance,
}).Error
if err != nil {
common.SysLog(fmt.Sprintf("failed to update balance: channel_id=%d, error=%v", channel.Id, err))
common.SysLog(common.LogText("failed to update balance: channel_id=%d, error=%v", channel.Id, err))
}
}
@@ -684,7 +689,7 @@ func handlerMultiKeyUpdate(channel *Channel, usingKey string, status int, reason
}
if keyIndex < 0 {
if usingKey != "" {
common.SysLog(fmt.Sprintf("failed to update multi-key status: channel_id=%d, using key not found", channel.Id))
common.SysLog(common.LogText("failed to update multi-key status: channel_id=%d, using key not found", channel.Id))
return
}
channel.Status = status
@@ -776,7 +781,7 @@ func UpdateChannelStatus(channelId int, usingKey string, status int, reason stri
if shouldUpdateAbilities {
err := UpdateAbilityStatus(channelId, status == common.ChannelStatusEnabled)
if err != nil {
common.SysLog(fmt.Sprintf("failed to update ability status: channel_id=%d, error=%v", channelId, err))
common.SysLog(common.LogText("failed to update ability status: channel_id=%d, error=%v", channelId, err))
}
}
}()
@@ -809,7 +814,7 @@ func UpdateChannelStatus(channelId int, usingKey string, status int, reason stri
}
err = channel.saveStatusState()
if err != nil {
common.SysLog(fmt.Sprintf("failed to update channel status: channel_id=%d, status=%d, error=%v", channel.Id, status, err))
common.SysLog(common.LogText("failed to update channel status: channel_id=%d, status=%d, error=%v", channel.Id, status, err))
return false
}
}
@@ -890,7 +895,7 @@ func EditChannelByTag(tag string, newTag *string, modelMapping *string, models *
for _, channel := range channels {
err = channel.UpdateAbilities(nil)
if err != nil {
common.SysLog(fmt.Sprintf("failed to update abilities: channel_id=%d, tag=%s, error=%v", channel.Id, channel.GetTag(), err))
common.SysLog(common.LogText("failed to update abilities: channel_id=%d, tag=%s, error=%v", channel.Id, channel.GetTag(), err))
}
}
}
@@ -914,7 +919,7 @@ func UpdateChannelUsedQuota(id int, quota int) {
func updateChannelUsedQuota(id int, quota int) {
err := DB.Model(&Channel{}).Where("id = ?", id).Update("used_quota", gorm.Expr("used_quota + ?", quota)).Error
if err != nil {
common.SysLog(fmt.Sprintf("failed to update channel used quota: channel_id=%d, delta_quota=%d, error=%v", id, quota, err))
common.SysLog(common.LogText("failed to update channel used quota: channel_id=%d, delta_quota=%d, error=%v", id, quota, err))
}
}
@@ -1040,7 +1045,7 @@ func (channel *Channel) GetSetting() dto.ChannelSettings {
if channel.Setting != nil && *channel.Setting != "" {
err := common.Unmarshal([]byte(*channel.Setting), &setting)
if err != nil {
common.SysLog(fmt.Sprintf("failed to unmarshal setting: channel_id=%d, error=%v", channel.Id, err))
common.SysLog(common.LogText("failed to unmarshal setting: channel_id=%d, error=%v", channel.Id, err))
channel.Setting = nil // 清空设置以避免后续错误
_ = channel.Save() // 保存修改
}
@@ -1051,7 +1056,7 @@ func (channel *Channel) GetSetting() dto.ChannelSettings {
func (channel *Channel) SetSetting(setting dto.ChannelSettings) {
settingBytes, err := common.Marshal(setting)
if err != nil {
common.SysLog(fmt.Sprintf("failed to marshal setting: channel_id=%d, error=%v", channel.Id, err))
common.SysLog(common.LogText("failed to marshal setting: channel_id=%d, error=%v", channel.Id, err))
return
}
channel.Setting = common.GetPointer[string](string(settingBytes))
@@ -1062,7 +1067,7 @@ func (channel *Channel) GetOtherSettings() dto.ChannelOtherSettings {
if channel.OtherSettings != "" {
err := common.UnmarshalJsonStr(channel.OtherSettings, &setting)
if err != nil {
common.SysLog(fmt.Sprintf("failed to unmarshal setting: channel_id=%d, error=%v", channel.Id, err))
common.SysLog(common.LogText("failed to unmarshal setting: channel_id=%d, error=%v", channel.Id, err))
channel.OtherSettings = "{}" // 清空设置以避免后续错误
_ = channel.Save() // 保存修改
}
@@ -1076,7 +1081,7 @@ func (channel *Channel) GetOtherSettings() dto.ChannelOtherSettings {
func (channel *Channel) SetOtherSettings(setting dto.ChannelOtherSettings) {
settingBytes, err := common.Marshal(setting)
if err != nil {
common.SysLog(fmt.Sprintf("failed to marshal setting: channel_id=%d, error=%v", channel.Id, err))
common.SysLog(common.LogText("failed to marshal setting: channel_id=%d, error=%v", channel.Id, err))
return
}
channel.OtherSettings = string(settingBytes)
@@ -1087,7 +1092,7 @@ func (channel *Channel) GetParamOverride() map[string]any {
if channel.ParamOverride != nil && *channel.ParamOverride != "" {
err := common.Unmarshal([]byte(*channel.ParamOverride), &paramOverride)
if err != nil {
common.SysLog(fmt.Sprintf("failed to unmarshal param override: channel_id=%d, error=%v", channel.Id, err))
common.SysLog(common.LogText("failed to unmarshal param override: channel_id=%d, error=%v", channel.Id, err))
}
}
return paramOverride
@@ -1098,7 +1103,7 @@ func (channel *Channel) GetHeaderOverride() map[string]any {
if channel.HeaderOverride != nil && *channel.HeaderOverride != "" {
err := common.Unmarshal([]byte(*channel.HeaderOverride), &headerOverride)
if err != nil {
common.SysLog(fmt.Sprintf("failed to unmarshal header override: channel_id=%d, error=%v", channel.Id, err))
common.SysLog(common.LogText("failed to unmarshal header override: channel_id=%d, error=%v", channel.Id, err))
}
}
return headerOverride
+7 -7
View File
@@ -108,13 +108,13 @@ func InitChannelCache() {
// invalidating the pricing cache, otherwise the reversed order deadlocks.
InvalidatePricingCache()
rebuildTaskAliasView()
common.SysLog("channels synced from database")
common.SysLog(common.LogText("channels synced from database"))
}
func SyncChannelCache(frequency int) {
for {
time.Sleep(time.Duration(frequency) * time.Second)
common.SysLog("syncing channels from database")
common.SysLog(common.LogText("syncing channels from database"))
InitChannelCache()
}
}
@@ -159,7 +159,7 @@ func GetRandomSatisfiedChannel(
if channel, ok := channelsIDM[channels[0]]; ok {
return channel, nil
}
return nil, fmt.Errorf("数据库一致性错误,渠道# %d 不存在,请联系管理员修复", channels[0])
return nil, fmt.Errorf("database consistency error: channel #%d does not exist, please contact the administrator", channels[0])
}
uniquePriorities := make(map[int]bool)
@@ -167,7 +167,7 @@ func GetRandomSatisfiedChannel(
if channel, ok := channelsIDM[channelId]; ok {
uniquePriorities[int(channel.GetPriority())] = true
} else {
return nil, fmt.Errorf("数据库一致性错误,渠道# %d 不存在,请联系管理员修复", channelId)
return nil, fmt.Errorf("database consistency error: channel #%d does not exist, please contact the administrator", channelId)
}
}
var sortedUniquePriorities []int
@@ -191,7 +191,7 @@ func GetRandomSatisfiedChannel(
targetChannels = append(targetChannels, channel)
}
} else {
return nil, fmt.Errorf("数据库一致性错误,渠道# %d 不存在,请联系管理员修复", channelId)
return nil, fmt.Errorf("database consistency error: channel #%d does not exist, please contact the administrator", channelId)
}
}
@@ -239,7 +239,7 @@ func CacheGetChannel(id int) (*Channel, error) {
c, ok := channelsIDM[id]
if !ok {
return nil, fmt.Errorf("渠道# %d,已不存在", id)
return nil, fmt.Errorf("channel #%d no longer exists", id)
}
return c, nil
}
@@ -257,7 +257,7 @@ func CacheGetChannelInfo(id int) (*ChannelInfo, error) {
c, ok := channelsIDM[id]
if !ok {
return nil, fmt.Errorf("渠道# %d,已不存在", id)
return nil, fmt.Errorf("channel #%d no longer exists", id)
}
return &c.ChannelInfo, nil
}
+9 -4
View File
@@ -123,11 +123,16 @@ func channelMatchesFilter(ch *Channel, modelName string, filter dto.ChannelFilte
config := ch.GetOtherSettings().AdvancedCustom
return config != nil && config.SupportsPathForModel(filter.RequestPath, modelName)
case dto.FilterTaskPluginIdentity:
if ch.Type == constant.ChannelTypeTaskPlugin {
key := ch.GetSetting().TaskPluginKey
return filter.TaskPluginKey != "" && (key == filter.TaskPluginKey || slices.Contains(filter.TaskPluginKeys, key))
if filter.TaskPluginKey == "" {
return ch.Type != constant.ChannelTypeTaskPlugin
}
return filter.TaskPluginKey == "" || slices.Contains(filter.TaskPluginChannelTypes, ch.Type)
if ch.Type == constant.ChannelTypeTaskPlugin || ch.Type == constant.ChannelTypeNewAPI {
// A New API channel serves every plugin it is extended with; the
// pinned plugin or any shared-model candidate may execute there.
setting := ch.GetSetting()
return setting.BindsTaskPlugin(filter.TaskPluginKey) || slices.ContainsFunc(filter.TaskPluginKeys, setting.BindsTaskPlugin)
}
return slices.Contains(filter.TaskPluginChannelTypes, ch.Type)
case dto.FilterResponsesWebSocket:
if !ch.GetSetting().ResponsesWebSocketEnabled {
return false
+244
View File
@@ -0,0 +1,244 @@
package model
import (
"crypto/sha256"
"errors"
"fmt"
"slices"
"github.com/QuantumNous/new-api/common"
"gorm.io/gorm"
)
// Keep priorities lossless in the web console's JSON number representation.
const maxSortablePriority int64 = 1<<53 - 1
var ErrChannelPriorityConflict = errors.New("Channel priorities changed. Refresh and try again.")
var ErrChannelPriorityInvalid = errors.New("Invalid channel priority move")
var ErrChannelPriorityRange = errors.New("Channel priority is outside the sortable range")
type ChannelPriority struct {
ID int `json:"id"`
Priority *int64 `json:"priority"`
}
func (item ChannelPriority) value() int64 {
if item.Priority == nil {
return 0
}
return *item.Priority
}
type ChannelPriorityChange struct {
ID int `json:"id"`
Previous *int64 `json:"previous"`
Priority *int64 `json:"priority"`
}
type ChannelPriorityOrder struct {
Revision string `json:"revision"`
Items []ChannelPriority `json:"items"`
Changes []ChannelPriorityChange `json:"changes"`
}
type ChannelPriorityMove struct {
Revision string `json:"revision"`
ID int `json:"id"`
TargetID int `json:"target_id"`
Position string `json:"position"` // before / after use descending priority order
Preview bool `json:"preview"`
Restore []ChannelPriorityChange `json:"restore,omitempty"`
}
func channelPriorityRevision(items []ChannelPriority) string {
data, _ := common.Marshal(items)
return fmt.Sprintf("%x", sha256.Sum256(data))
}
func GetChannelPriorityOrder() (ChannelPriorityOrder, error) {
result := ChannelPriorityOrder{Items: []ChannelPriority{}, Changes: []ChannelPriorityChange{}}
err := DB.Model(&Channel{}).Select("id", "priority").Order("id").Find(&result.Items).Error
result.Revision = channelPriorityRevision(result.Items)
return result, err
}
// ReorderChannelPriority updates only priorities, including the routing copy in
// abilities. A snapshot check also protects preview/commit and undo from stale
// clients; row locks serialize concurrent reorders on MySQL and PostgreSQL.
func ReorderChannelPriority(move ChannelPriorityMove) (ChannelPriorityOrder, error) {
result := ChannelPriorityOrder{Items: []ChannelPriority{}, Changes: []ChannelPriorityChange{}}
err := DB.Transaction(func(tx *gorm.DB) error {
query := tx.Model(&Channel{}).Select("id", "priority").Order("id")
if !move.Preview {
query = lockForUpdate(query)
}
if err := query.Find(&result.Items).Error; err != nil {
return err
}
if move.Revision == "" || move.Revision != channelPriorityRevision(result.Items) {
return ErrChannelPriorityConflict
}
changes, err := planChannelPriorityMove(result.Items, move)
if err != nil {
return err
}
result.Changes = changes
byID := make(map[int]*ChannelPriority, len(result.Items))
for i := range result.Items {
byID[result.Items[i].ID] = &result.Items[i]
}
for _, change := range changes {
byID[change.ID].Priority = change.Priority
if move.Preview {
continue
}
if err := tx.Model(&Channel{}).Where("id = ?", change.ID).Update("priority", change.Priority).Error; err != nil {
return err
}
if err := tx.Model(&Ability{}).Where("channel_id = ?", change.ID).Update("priority", change.Priority).Error; err != nil {
return err
}
}
result.Revision = channelPriorityRevision(result.Items)
return nil
})
return result, err
}
// Equal-priority channels remain together. Opening a slot shifts the shorter
// contiguous run of occupied integer priorities, measured by affected channels.
func planChannelPriorityMove(items []ChannelPriority, move ChannelPriorityMove) ([]ChannelPriorityChange, error) {
changes := []ChannelPriorityChange{}
byID := make(map[int]ChannelPriority, len(items))
for _, item := range items {
if item.value() < -maxSortablePriority || item.value() > maxSortablePriority {
return nil, ErrChannelPriorityRange
}
byID[item.ID] = item
}
if len(move.Restore) > 0 {
if move.ID != 0 || move.TargetID != 0 || move.Position != "" {
return nil, ErrChannelPriorityInvalid
}
seen := make(map[int]bool, len(move.Restore))
for _, change := range move.Restore {
item, ok := byID[change.ID]
if !ok || seen[change.ID] || (change.Priority != nil && (*change.Priority < -maxSortablePriority || *change.Priority > maxSortablePriority)) {
return nil, ErrChannelPriorityInvalid
}
if !equalPriority(item.Priority, change.Previous) {
return nil, ErrChannelPriorityConflict
}
seen[change.ID] = true
if !equalPriority(item.Priority, change.Priority) {
changes = append(changes, change)
}
}
return changes, nil
}
item, ok := byID[move.ID]
target, targetOK := byID[move.TargetID]
if !ok || !targetOK || move.ID == move.TargetID || !slices.Contains([]string{"before", "after", "same"}, move.Position) {
return nil, ErrChannelPriorityInvalid
}
values := make(map[int]int64, len(items))
for _, entry := range items {
values[entry.ID] = entry.value()
}
if move.Position == "same" {
values[item.ID] = target.value()
} else {
counts := make(map[int64]int)
for _, entry := range items {
if entry.ID != item.ID {
counts[entry.value()]++
}
}
levels := make([]int64, 0, len(counts))
for level := range counts {
levels = append(levels, level)
}
slices.SortFunc(levels, func(a, b int64) int {
if a > b {
return -1
}
if a < b {
return 1
}
return 0
})
index := slices.Index(levels, target.value())
if move.Position == "after" {
index++
}
hi, lo := maxSortablePriority+1, -maxSortablePriority-1
if index > 0 {
hi = levels[index-1]
}
if index < len(levels) {
lo = levels[index]
}
// Dropping into the channel's existing slot must not renumber it.
if item.value() < hi && item.value() > lo {
return changes, nil
}
if hi-lo < 2 {
up, down := index-1, index
upCost, downCost := 0, 0
for up >= 0 {
upCost += counts[levels[up]]
if (up == 0 && levels[up] < maxSortablePriority) || (up > 0 && levels[up-1]-levels[up] > 1) {
break
}
up--
}
for down < len(levels) {
downCost += counts[levels[down]]
if (down == len(levels)-1 && levels[down] > -maxSortablePriority) || (down+1 < len(levels) && levels[down]-levels[down+1] > 1) {
break
}
down++
}
canUp, canDown := up >= 0 && index > 0, down < len(levels) && index < len(levels)
if !canUp && !canDown {
return nil, ErrChannelPriorityRange
}
if canUp && (!canDown || upCost < downCost) {
for id, value := range values {
if id != item.ID && value >= levels[index-1] && value <= levels[up] {
values[id]++
}
}
hi++
} else {
for id, value := range values {
if id != item.ID && value <= levels[index] && value >= levels[down] {
values[id]--
}
}
lo--
}
}
value := lo + (hi-lo)/2
if index == 0 {
value = min(maxSortablePriority, lo+10)
} else if index == len(levels) {
value = max(-maxSortablePriority, hi-10)
}
values[item.ID] = value
}
for _, entry := range items {
value := values[entry.ID]
if value != entry.value() {
changes = append(changes, ChannelPriorityChange{ID: entry.ID, Previous: entry.Priority, Priority: &value})
}
}
return changes, nil
}
func equalPriority(a, b *int64) bool {
if a == nil || b == nil {
return a == nil && b == nil
}
return *a == *b
}
+209
View File
@@ -0,0 +1,209 @@
package model
import (
"errors"
"os"
"sync"
"testing"
"github.com/QuantumNous/new-api/common"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/schema"
)
func TestChannelPriorityPlan(t *testing.T) {
for _, tc := range []struct {
name string
values []int64
id, target int
position string
want map[int]int64
}{
{"midpoint", []int64{100, 80, 0}, 3, 2, "before", map[int]int64{3: 90}},
{"top preserves peers", []int64{0, 0, 0}, 3, 1, "before", map[int]int64{3: 10}},
{"bottom", []int64{0, 0, 0}, 1, 3, "after", map[int]int64{1: -10}},
{"same tier", []int64{100, 80, 0}, 3, 2, "same", map[int]int64{3: 80}},
{"same slot", []int64{100, 80, 0}, 2, 1, "after", map[int]int64{}},
{"same tier no op", []int64{0, 0}, 1, 2, "same", map[int]int64{}},
{"shift smaller upper tier", []int64{100, 99, 99, 0}, 4, 2, "before", map[int]int64{1: 101, 4: 100}},
{"shift smaller lower tier", []int64{100, 100, 99, 0}, 4, 3, "before", map[int]int64{3: 98, 4: 99}},
{"shift complete adjacent tiers", []int64{102, 101, 100, 99, 0}, 5, 3, "after", map[int]int64{4: 98, 5: 99}},
{"upper safe bound", []int64{maxSortablePriority, 0}, 2, 1, "before", map[int]int64{1: maxSortablePriority - 1, 2: maxSortablePriority}},
{"lower safe bound", []int64{-maxSortablePriority, 0}, 2, 1, "after", map[int]int64{1: -maxSortablePriority + 1, 2: -maxSortablePriority}},
} {
t.Run(tc.name, func(t *testing.T) {
items := make([]ChannelPriority, len(tc.values))
for i, value := range tc.values {
items[i] = ChannelPriority{ID: i + 1, Priority: &value}
}
changes, err := planChannelPriorityMove(items, ChannelPriorityMove{ID: tc.id, TargetID: tc.target, Position: tc.position})
require.NoError(t, err)
got := map[int]int64{}
for _, change := range changes {
require.NotNil(t, change.Priority)
got[change.ID] = *change.Priority
}
assert.Equal(t, tc.want, got)
})
}
_, err := planChannelPriorityMove([]ChannelPriority{{ID: 1}, {ID: 2}}, ChannelPriorityMove{ID: 1, TargetID: 2, Position: "invalid"})
assert.ErrorIs(t, err, ErrChannelPriorityInvalid)
}
func TestChannelPriorityDatabaseMatrix(t *testing.T) {
for _, dialect := range []string{"sqlite", "mysql", "postgres"} {
t.Run(dialect, func(t *testing.T) {
var driver gorm.Dialector
switch dialect {
case "sqlite":
driver = sqlite.Open(":memory:")
case "mysql":
dsn := os.Getenv("TEST_MYSQL_DSN")
if dsn == "" {
t.Skip("TEST_MYSQL_DSN not configured")
}
driver = mysql.Open(dsn)
case "postgres":
dsn := os.Getenv("TEST_POSTGRES_DSN")
if dsn == "" {
t.Skip("TEST_POSTGRES_DSN not configured")
}
driver = postgres.Open(dsn)
}
db, err := gorm.Open(driver, &gorm.Config{NamingStrategy: schema.NamingStrategy{TablePrefix: "priority_test_"}})
require.NoError(t, err)
sqlDB, err := db.DB()
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
previousDB, previousType := DB, common.MainDatabaseType()
DB = db
common.SetMainDatabaseType(common.DatabaseType(dialect))
initCol()
t.Cleanup(func() {
require.NoError(t, db.Migrator().DropTable(&Ability{}, &Channel{}))
DB = previousDB
common.SetMainDatabaseType(previousType)
initCol()
require.NoError(t, sqlDB.Close())
})
require.NoError(t, db.AutoMigrate(&Channel{}, &Ability{}))
var version string
query := "SELECT version()"
if dialect == "sqlite" {
query = "SELECT sqlite_version()"
}
require.NoError(t, db.Raw(query).Scan(&version).Error)
t.Logf("database version: %s", version)
high, low, zero := int64(100), int64(99), int64(0)
w1, w2, w3, w4 := uint(11), uint(22), uint(33), uint(44)
channels := []Channel{
{Name: "high", Priority: &high, Weight: &w1, Key: "preserved", Status: 1},
{Name: "low", Priority: &low, Weight: &w2, Status: 1},
{Name: "peer", Priority: &low, Weight: &w3, Status: 1},
{Name: "moving", Priority: &zero, Weight: &w4, Status: 1},
}
require.NoError(t, db.Create(&channels).Error)
for _, channel := range channels {
for _, group := range []string{"default", "second"} {
require.NoError(t, db.Create(&Ability{Group: group, Model: "test", ChannelId: channel.Id, Priority: channel.Priority, Weight: uint(*channel.Weight), Enabled: true}).Error)
}
}
initial, err := GetChannelPriorityOrder()
require.NoError(t, err)
move := ChannelPriorityMove{Revision: initial.Revision, ID: channels[3].Id, TargetID: channels[1].Id, Position: "before", Preview: true}
preview, err := ReorderChannelPriority(move)
require.NoError(t, err)
require.Len(t, preview.Changes, 2)
unchanged, err := GetChannelPriorityOrder()
require.NoError(t, err)
assert.Equal(t, initial, unchanged, "preview does not persist")
move.Preview = false
applied, err := ReorderChannelPriority(move)
require.NoError(t, err)
assert.Equal(t, preview, applied)
for _, change := range applied.Changes {
var abilities []Ability
require.NoError(t, db.Where("channel_id = ?", change.ID).Find(&abilities).Error)
require.Len(t, abilities, 2)
for _, ability := range abilities {
assert.Equal(t, change.Priority, ability.Priority)
assert.True(t, ability.Enabled)
}
}
var saved Channel
require.NoError(t, db.First(&saved, channels[0].Id).Error)
assert.Equal(t, "preserved", saved.Key)
assert.Equal(t, channels[0].Weight, saved.Weight)
_, err = ReorderChannelPriority(move)
assert.ErrorIs(t, err, ErrChannelPriorityConflict)
undo := ChannelPriorityMove{Revision: applied.Revision}
for _, change := range applied.Changes {
undo.Restore = append(undo.Restore, ChannelPriorityChange{ID: change.ID, Previous: change.Priority, Priority: change.Previous})
}
restored, err := ReorderChannelPriority(undo)
require.NoError(t, err)
assert.Equal(t, initial.Items, restored.Items)
assert.Equal(t, initial.Revision, restored.Revision)
if dialect != "sqlite" {
sqlDB.SetMaxOpenConns(2)
}
start := make(chan struct{})
errorsCh := make(chan error, 2)
var workers sync.WaitGroup
for range 2 {
workers.Go(func() { <-start; _, moveErr := ReorderChannelPriority(move); errorsCh <- moveErr })
}
close(start)
workers.Wait()
close(errorsCh)
succeeded, conflicted := 0, 0
for moveErr := range errorsCh {
if moveErr == nil {
succeeded++
} else {
require.ErrorIs(t, moveErr, ErrChannelPriorityConflict)
conflicted++
}
}
assert.Equal(t, 1, succeeded)
assert.Equal(t, 1, conflicted)
_, err = ReorderChannelPriority(undo)
require.NoError(t, err)
sqlDB.SetMaxOpenConns(1)
// An ability write failure must roll back every channel change too.
require.NoError(t, db.Callback().Update().Before("gorm:update").Register("priority_test_failure", func(tx *gorm.DB) {
if tx.Statement.Table == "priority_test_abilities" {
tx.AddError(errors.New("forced routing write failure"))
}
}))
_, err = ReorderChannelPriority(move)
require.ErrorContains(t, err, "forced routing write failure")
require.NoError(t, db.Callback().Update().Remove("priority_test_failure"))
unchanged, err = GetChannelPriorityOrder()
require.NoError(t, err)
assert.Equal(t, initial, unchanged)
// Null priority is an effective zero and undo must restore the null itself.
require.NoError(t, db.Model(&Channel{}).Where("id = ?", channels[3].Id).Update("priority", nil).Error)
nullable, err := GetChannelPriorityOrder()
require.NoError(t, err)
var sorted []ChannelPriority
require.NoError(t, NewChannelSortOptions("priority", "asc", false).Apply(db.Model(&Channel{}).Select("id", "priority")).Find(&sorted).Error)
assert.Equal(t, []int{channels[3].Id, channels[2].Id, channels[1].Id, channels[0].Id}, []int{sorted[0].ID, sorted[1].ID, sorted[2].ID, sorted[3].ID})
move.Revision = nullable.Revision
applied, err = ReorderChannelPriority(move)
require.NoError(t, err)
undo = ChannelPriorityMove{Revision: applied.Revision}
for _, change := range applied.Changes {
undo.Restore = append(undo.Restore, ChannelPriorityChange{ID: change.ID, Previous: change.Priority, Priority: change.Previous})
}
restored, err = ReorderChannelPriority(undo)
require.NoError(t, err)
assert.Equal(t, nullable.Items, restored.Items)
})
}
}
+6 -7
View File
@@ -1,7 +1,6 @@
package model
import (
"errors"
"math/rand"
"time"
@@ -55,7 +54,7 @@ func HasCheckedInToday(userId int) (bool, error) {
func UserCheckin(userId int) (*Checkin, error) {
setting := operation_setting.GetCheckinSetting()
if !setting.Enabled {
return nil, errors.New("签到功能未启用")
return nil, common.NewMessage("Check-in feature is not enabled")
}
// 检查今天是否已签到
@@ -64,7 +63,7 @@ func UserCheckin(userId int) (*Checkin, error) {
return nil, err
}
if hasChecked {
return nil, errors.New("今日已签到")
return nil, common.NewMessage("Already checked in today")
}
// 计算随机额度奖励
@@ -97,13 +96,13 @@ func userCheckinWithTransaction(checkin *Checkin, userId int, quotaAwarded int)
// 步骤1: 创建签到记录
// 数据库有唯一约束 (user_id, checkin_date),可以防止并发重复签到
if err := tx.Create(checkin).Error; err != nil {
return errors.New("签到失败,请稍后重试")
return common.NewMessage("Check-in failed, please try again later")
}
// 步骤2: 在事务中增加用户额度
if err := tx.Model(&User{}).Where("id = ?", userId).
Update("quota", gorm.Expr("quota + ?", quotaAwarded)).Error; err != nil {
return errors.New("签到失败:更新额度出错")
return common.NewMessage("Check-in failed: quota update error")
}
return nil
@@ -126,7 +125,7 @@ func userCheckinWithoutTransaction(checkin *Checkin, userId int, quotaAwarded in
// 步骤1: 创建签到记录
// 数据库有唯一约束 (user_id, checkin_date),可以防止并发重复签到
if err := DB.Create(checkin).Error; err != nil {
return nil, errors.New("签到失败,请稍后重试")
return nil, common.NewMessage("Check-in failed, please try again later")
}
// 步骤2: 增加用户额度
@@ -134,7 +133,7 @@ func userCheckinWithoutTransaction(checkin *Checkin, userId int, quotaAwarded in
if err := IncreaseUserQuota(userId, quotaAwarded, true); err != nil {
// 如果增加额度失败,需要回滚签到记录
DB.Delete(checkin)
return nil, errors.New("签到失败:更新额度出错")
return nil, common.NewMessage("Check-in failed: quota update error")
}
return checkin, nil
+14 -13
View File
@@ -39,18 +39,19 @@ type DetectionRun struct {
}
type DetectionCall struct {
ID string `gorm:"primaryKey;type:varchar(64)"`
RunID string `gorm:"index;type:varchar(64)"`
SecretHash string `gorm:"uniqueIndex;type:varchar(64)"`
ExpiresAt int64
StartedAt int64
FinishedAt int64
State string `gorm:"type:varchar(24)"`
Usage string `gorm:"type:text"`
Sample string `gorm:"type:text"`
Quota int
Available *bool
Error string
FirstTokenMS *int64
ID string `gorm:"primaryKey;type:varchar(64)"`
RunID string `gorm:"index;type:varchar(64)"`
SecretHash string `gorm:"uniqueIndex;type:varchar(64)"`
ExpiresAt int64
StartedAt int64
FinishedAt int64
State string `gorm:"type:varchar(24)"`
Usage string `gorm:"type:text"`
Sample string `gorm:"type:text"`
Quota int
Available *bool
Error string
}
func DetectionSecretHash(secret string) string {
@@ -144,7 +145,7 @@ func FinishDetectionCall(call *DetectionCall) error {
return errors.New("invalid charge")
}
return DB.Model(&DetectionCall{}).Where("id = ? AND state = ?", call.ID, "running").Updates(map[string]any{
"state": "finished", "finished_at": time.Now().Unix(), "usage": call.Usage, "sample": call.Sample, "quota": call.Quota, "available": call.Available, "error": call.Error,
"state": "finished", "finished_at": time.Now().Unix(), "usage": call.Usage, "sample": call.Sample, "quota": call.Quota, "available": call.Available, "error": call.Error, "first_token_ms": call.FirstTokenMS,
}).Error
}
+5 -2
View File
@@ -20,7 +20,7 @@ type DetectionSchedule struct {
}
func NextDetectionInterval(lastValid, outcome string, current int) int {
if outcome != "normal" && outcome != "degraded" {
if outcome != "normal" && outcome != "degraded" && outcome != "incomplete" {
return 15
}
if lastValid == "" {
@@ -51,7 +51,10 @@ func ApplyDetectionScheduleResult(ctx context.Context, channelID int, kind, resu
return nil
}
state.IntervalMinutes = NextDetectionInterval(state.LastValid, outcome, state.IntervalMinutes)
if outcome == "normal" || outcome == "degraded" {
if kind == "ciallo" {
state.IntervalMinutes = 45
}
if outcome == "normal" || outcome == "degraded" || outcome == "incomplete" {
state.LastValid = outcome
}
state.LastFinishedAt, state.LastResultID = finishedAt, resultID
+10 -2
View File
@@ -26,10 +26,11 @@ func TestAdaptiveDetectionIntervals(t *testing.T) {
}{
{"normal", 30}, {"normal", 60}, {"normal", 60}, {"degraded", 15}, {"normal", 15}, {"normal", 30}, {"normal", 60},
{"unknown", 15}, {"unknown", 15}, {"normal", 30}, {"normal", 60}, {"failed", 15}, {"degraded", 15}, {"degraded", 30}, {"degraded", 60},
{"incomplete", 15}, {"incomplete", 30}, {"incomplete", 60}, {"failed", 15}, {"incomplete", 30}, {"normal", 15},
} {
state.IntervalMinutes = NextDetectionInterval(state.LastValid, step.result, state.IntervalMinutes)
assert.Equal(t, step.interval, state.IntervalMinutes, step.result)
if step.result == "normal" || step.result == "degraded" {
if step.result == "normal" || step.result == "degraded" || step.result == "incomplete" {
state.LastValid = step.result
}
}
@@ -119,6 +120,13 @@ func TestAdaptiveDetectionDatabaseMatrix(t *testing.T) {
require.NoError(t, db.First(&loaded, "channel_id = ? AND kind = ?", 12, "pelican").Error)
assert.Equal(t, 15, loaded.IntervalMinutes)
assert.Equal(t, int64(6400), loaded.NextDueAt)
require.NoError(t, EnsureDetectionSchedule(ctx, DetectionSchedule{ChannelID: 13, Kind: "vision", IntervalMinutes: 30}))
require.NoError(t, ApplyDetectionScheduleResult(ctx, 13, "vision", "incomplete", 7000, "incomplete"))
require.NoError(t, ApplyDetectionScheduleResult(ctx, 13, "vision", "transport-error", 7100, "failed"))
var visionSchedule DetectionSchedule
require.NoError(t, db.First(&visionSchedule, "channel_id = ? AND kind = ?", 13, "vision").Error)
assert.Equal(t, "incomplete", visionSchedule.LastValid)
assert.Equal(t, 15, visionSchedule.IntervalMinutes)
assert.Equal(t, "degraded", loaded.LastValid)
seed.Kind = "fingerprint"
require.NoError(t, EnsureDetectionSchedule(ctx, seed))
@@ -127,7 +135,7 @@ func TestAdaptiveDetectionDatabaseMatrix(t *testing.T) {
require.NoError(t, db.AutoMigrate(&SystemTask{}, &DetectionSchedule{}))
var rows []DetectionSchedule
require.NoError(t, db.Find(&rows).Error)
require.Len(t, rows, 2)
require.Len(t, rows, 3)
// Reinitializing from history must not overwrite a persisted clock.
seed.Kind = "pelican"
require.NoError(t, EnsureDetectionSchedule(ctx, seed))
+3 -3
View File
@@ -85,7 +85,7 @@ func migrateLegacyOption(sourceKey, targetKey string, transform legacyOptionTran
value, transformErr := transform(source.Value)
if transformErr != nil {
common.SysError(fmt.Sprintf("legacy option %s was not migrated: %v", sourceKey, transformErr))
common.SysError(common.LogText("legacy option %s was not migrated: %v", sourceKey, transformErr))
return nil
}
if errors.Is(err, gorm.ErrRecordNotFound) {
@@ -205,7 +205,7 @@ func migrateLegacyUptimeOptions() error {
}
if urlErr != nil || slugErr != nil || strings.TrimSpace(urlOption.Value) == "" || strings.TrimSpace(slugOption.Value) == "" {
common.SysError("legacy Uptime Kuma options were not migrated: both URL and slug are required")
common.SysError(common.LogText("legacy Uptime Kuma options were not migrated: both URL and slug are required"))
return nil
}
groups := []map[string]any{{
@@ -221,7 +221,7 @@ func migrateLegacyUptimeOptions() error {
}
value := string(encoded)
if err := console_setting.ValidateConsoleSettings(value, "UptimeKumaGroups"); err != nil {
common.SysError(fmt.Sprintf("legacy Uptime Kuma options were not migrated: %v", err))
common.SysError(common.LogText("legacy Uptime Kuma options were not migrated: %v", err))
return nil
}
if errors.Is(targetErr, gorm.ErrRecordNotFound) {
+1 -1
View File
@@ -32,7 +32,7 @@ func newGormConfig(prepareStmt bool) *gorm.Config {
func newGormLogger(w io.Writer) logger.Interface {
slowThresholdMs := common.GetEnvOrDefault("SQL_SLOW_THRESHOLD_MS", defaultSlowThresholdMs)
if slowThresholdMs < 0 || slowThresholdMs > maxSlowThresholdMs {
common.SysError(fmt.Sprintf("invalid SQL_SLOW_THRESHOLD_MS %d (allowed 0-%d, 0 disables slow query log), using default %d", slowThresholdMs, maxSlowThresholdMs, defaultSlowThresholdMs))
common.SysError(common.LogText("invalid SQL_SLOW_THRESHOLD_MS %d (allowed 0-%d, 0 disables slow query log), using default %d", slowThresholdMs, maxSlowThresholdMs, defaultSlowThresholdMs))
slowThresholdMs = defaultSlowThresholdMs
}
// 在 Writer 层脱敏而非包装 logger.Interface:后者会让 gorm 的 FileWithLineNum
+179
View File
@@ -0,0 +1,179 @@
package model
import (
"context"
"math"
"github.com/QuantumNous/new-api/common"
)
type ModelPerformanceMetric struct {
Value float64 `json:"value"`
Samples int `json:"samples"`
Source string `json:"source"`
UpdatedAt int64 `json:"updated_at"`
}
type GroupModelPerformance struct {
Throughput *ModelPerformanceMetric `json:"throughput,omitempty"`
FirstToken *ModelPerformanceMetric `json:"first_token,omitempty"`
}
type performanceAccumulator struct {
total float64
count int
latest int64
}
func (a *performanceAccumulator) add(value *float64, at int64) {
if value == nil {
return
}
a.total += *value
a.count++
a.latest = max(a.latest, at)
}
func (a performanceAccumulator) metric(source string) *ModelPerformanceMetric {
if a.count == 0 {
return nil
}
return &ModelPerformanceMetric{Value: a.total / float64(a.count), Samples: a.count, Source: source, UpdatedAt: a.latest}
}
// The throughput definition matches usage logs: output tokens / total request
// seconds. New logs retain milliseconds; legacy logs use their stored seconds.
func modelLogPerformance(log Log) (throughput, firstToken *float64) {
var other struct {
FirstToken *float64 `json:"frt"`
LatencyMs *float64 `json:"latency_ms"`
StreamStatus struct {
Status string `json:"status"`
EndReason string `json:"end_reason"`
ResponseStatus string `json:"response_status"`
} `json:"stream_status"`
}
if common.UnmarshalJsonStr(log.Other, &other) != nil ||
(other.StreamStatus.Status != "" && other.StreamStatus.Status != "ok") ||
other.StreamStatus.EndReason == "client_gone" ||
(other.StreamStatus.ResponseStatus != "" && other.StreamStatus.ResponseStatus != "completed") {
return nil, nil
}
if log.IsStream && other.FirstToken != nil && *other.FirstToken >= 0 && !math.IsInf(*other.FirstToken, 0) && !math.IsNaN(*other.FirstToken) {
firstToken = other.FirstToken
}
duration := float64(log.UseTime) * 1000
if other.LatencyMs != nil {
duration = *other.LatencyMs
}
if log.CompletionTokens > 0 && duration > 0 && !math.IsInf(duration, 0) && !math.IsNaN(duration) {
value := float64(log.CompletionTokens) * 1000 / duration
if value > 0 && !math.IsInf(value, 0) {
throughput = &value
}
}
return throughput, firstToken
}
// Attribute calls to their billed group, never to a channel's current groups.
// Each metric keeps its latest thirty usable calls independently. Failed and
// non-streaming calls cannot replace a known first-token measurement.
func GetGroupModelPerformance(ctx context.Context, group, modelName string) (GroupModelPerformance, error) {
result := GroupModelPerformance{}
if LOG_DB == nil || group == "" || modelName == "" {
return result, nil
}
var throughput, firstToken performanceAccumulator
lastID, lastAt := int64(0), int64(0)
for {
var logs []Log
query := LOG_DB.WithContext(ctx).Model(&Log{}).
Select("id", "created_at", "completion_tokens", "use_time", "is_stream", "other").
Where(map[string]any{"type": LogTypeConsume, "group": group, "model_name": modelName}).
Where("token_id > 0").Order("created_at DESC, id DESC").Limit(30)
if throughput.count == 30 {
query = query.Where("is_stream = ?", true)
}
if lastID > 0 {
query = query.Where("created_at < ? OR (created_at = ? AND id < ?)", lastAt, lastAt, lastID)
}
if err := query.Find(&logs).Error; err != nil {
return result, err
}
for _, log := range logs {
tps, ttft := modelLogPerformance(log)
if throughput.count < 30 {
throughput.add(tps, log.CreatedAt)
}
if firstToken.count < 30 {
firstToken.add(ttft, log.CreatedAt)
}
}
if len(logs) < 30 || (throughput.count == 30 && firstToken.count == 30) {
break
}
lastID = int64(logs[len(logs)-1].Id)
lastAt = logs[len(logs)-1].CreatedAt
}
result.Throughput = throughput.metric("calls")
result.FirstToken = firstToken.metric("calls")
if result.Throughput != nil && result.FirstToken != nil {
return result, nil
}
var channelIDs []int
if err := DB.WithContext(ctx).Model(&Ability{}).
Where(map[string]any{"group": group, "model": modelName, "enabled": true}).
Pluck("channel_id", &channelIDs).Error; err != nil {
return result, err
}
if len(channelIDs) == 0 {
return result, nil
}
if err := DB.WithContext(ctx).Model(&Channel{}).
Where("id IN ? AND status = ?", channelIDs, common.ChannelStatusEnabled).
Pluck("id", &channelIDs).Error; err != nil {
return result, err
}
if len(channelIDs) == 0 {
return result, nil
}
rows, err := LOG_DB.WithContext(ctx).Model(&Log{}).
Select("id", "channel_id", "created_at", "completion_tokens", "use_time", "is_stream", "other").
Where("type = ? AND token_id = 0 AND model_name = ? AND is_stream = ? AND channel_id IN ?", LogTypeConsume, modelName, true, channelIDs).
Where("content IN ?", []string{"模型测试", "Model test"}).Order("created_at DESC, id DESC").Rows()
if err != nil {
return result, err
}
defer rows.Close()
var testThroughput, testFirstToken performanceAccumulator
tpsChannels, ttftChannels := map[int]bool{}, map[int]bool{}
for rows.Next() {
var log Log
if err := LOG_DB.ScanRows(rows, &log); err != nil {
return result, err
}
tps, ttft := modelLogPerformance(log)
if tps != nil && !tpsChannels[log.ChannelId] {
testThroughput.add(tps, log.CreatedAt)
tpsChannels[log.ChannelId] = true
}
if ttft != nil && !ttftChannels[log.ChannelId] {
testFirstToken.add(ttft, log.CreatedAt)
ttftChannels[log.ChannelId] = true
}
if (result.Throughput != nil || len(tpsChannels) == len(channelIDs)) &&
(result.FirstToken != nil || len(ttftChannels) == len(channelIDs)) {
break
}
}
if err := rows.Err(); err != nil {
return result, err
}
if result.Throughput == nil {
result.Throughput = testThroughput.metric("test")
}
if result.FirstToken == nil {
result.FirstToken = testFirstToken.metric("test")
}
return result, nil
}
+116
View File
@@ -0,0 +1,116 @@
package model
import (
"context"
"fmt"
"os"
"testing"
"time"
"github.com/QuantumNous/new-api/common"
"github.com/glebarez/sqlite"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"gorm.io/driver/mysql"
"gorm.io/driver/postgres"
"gorm.io/gorm"
"gorm.io/gorm/schema"
)
func TestGroupModelPerformanceDatabaseMatrix(t *testing.T) {
for _, dialect := range []string{"sqlite", "mysql", "postgres"} {
t.Run(dialect, func(t *testing.T) {
var driver gorm.Dialector
switch dialect {
case "sqlite":
driver = sqlite.Open(":memory:")
case "mysql":
if os.Getenv("TEST_MYSQL_DSN") == "" {
t.Skip("TEST_MYSQL_DSN unset")
}
driver = mysql.Open(os.Getenv("TEST_MYSQL_DSN"))
case "postgres":
if os.Getenv("TEST_POSTGRES_DSN") == "" {
t.Skip("TEST_POSTGRES_DSN unset")
}
driver = postgres.Open(os.Getenv("TEST_POSTGRES_DSN"))
}
db, err := gorm.Open(driver, &gorm.Config{NamingStrategy: schema.NamingStrategy{TablePrefix: fmt.Sprintf("performance_%d_", time.Now().UnixNano())}})
require.NoError(t, err)
conn, err := db.DB()
require.NoError(t, err)
conn.SetMaxOpenConns(1)
previousDB, previousLogs := DB, LOG_DB
DB, LOG_DB = db, db
t.Cleanup(func() {
for _, table := range []any{&Log{}, &Ability{}, &Channel{}} {
require.NoError(t, db.Migrator().DropTable(table))
}
DB, LOG_DB = previousDB, previousLogs
require.NoError(t, conn.Close())
})
query := "select version()"
if dialect == "sqlite" {
query = "select sqlite_version()"
}
var version string
require.NoError(t, db.Raw(query).Scan(&version).Error)
t.Logf("database version: %s", version)
require.NoError(t, db.AutoMigrate(&Log{}, &Ability{}, &Channel{}))
require.NoError(t, db.Create(&Channel{Id: 1, Status: common.ChannelStatusEnabled, Group: "primary,backup"}).Error)
require.NoError(t, db.Create(&Channel{Id: 2, Status: common.ChannelStatusEnabled, Group: "primary"}).Error)
require.NoError(t, db.Create(&Channel{Id: 3, Status: common.ChannelStatusAutoDisabled, Group: "primary"}).Error)
for _, channel := range []int{1, 2, 3} {
require.NoError(t, db.Create(&Ability{ChannelId: channel, Group: "primary", Model: "m", Enabled: true}).Error)
}
ctx := context.Background()
empty, err := GetGroupModelPerformance(ctx, "primary", "m")
require.NoError(t, err)
assert.Nil(t, empty.Throughput)
assert.Nil(t, empty.FirstToken)
logs := []Log{
{ChannelId: 1, Type: LogTypeConsume, Content: "模型测试", ModelName: "m", IsStream: true, CreatedAt: 1, UseTime: 10, CompletionTokens: 100, Other: `{"frt":1000}`},
{ChannelId: 1, Type: LogTypeConsume, Content: "Model test", ModelName: "m", IsStream: true, CreatedAt: 2, UseTime: 1, CompletionTokens: 20, Other: `{"frt":500,"latency_ms":500}`},
{ChannelId: 2, Type: LogTypeConsume, Content: "Model test", ModelName: "m", IsStream: true, CreatedAt: 2, UseTime: 10, CompletionTokens: 200, Other: `{"frt":1500}`},
{ChannelId: 3, Type: LogTypeConsume, Content: "Model test", ModelName: "m", IsStream: true, CreatedAt: 3, UseTime: 1, CompletionTokens: 999, Other: `{"frt":1}`},
{ChannelId: 1, Type: LogTypeConsume, Content: "Model test", ModelName: "other", IsStream: true, CreatedAt: 3, UseTime: 1, CompletionTokens: 999, Other: `{"frt":1}`},
{ChannelId: 1, Type: LogTypeConsume, Content: "Model test", ModelName: "m", IsStream: true, CreatedAt: 4, UseTime: 1, CompletionTokens: 999, Other: `{"frt":1,"stream_status":{"status":"error"}}`},
}
require.NoError(t, db.Create(&logs).Error)
fallback, err := GetGroupModelPerformance(ctx, "primary", "m")
require.NoError(t, err)
require.NotNil(t, fallback.Throughput)
assert.Equal(t, ModelPerformanceMetric{Value: 30, Samples: 2, Source: "test", UpdatedAt: 2}, *fallback.Throughput)
assert.Equal(t, 1000.0, fallback.FirstToken.Value)
calls := make([]Log, 0, 31)
for i := range 31 {
calls = append(calls, Log{ChannelId: 1, Type: LogTypeConsume, TokenId: 8, Group: "primary", ModelName: "m", IsStream: true, CreatedAt: int64(100 + i), UseTime: i + 1, CompletionTokens: (i + 1) * (i + 1), Other: fmt.Sprintf(`{"frt":%d}`, (i+1)*100)})
}
require.NoError(t, db.Create(&calls).Error)
// Invalid, another group's, and synthetic requests must not displace real samples.
noise := []Log{
{Type: LogTypeConsume, TokenId: 8, Group: "backup", ModelName: "m", IsStream: true, CreatedAt: 999, UseTime: 1, CompletionTokens: 999, Other: `{"frt":1}`},
{Type: LogTypeConsume, TokenId: 8, Group: "primary", ModelName: "other", IsStream: true, CreatedAt: 999, UseTime: 1, CompletionTokens: 999, Other: `{"frt":1}`},
{Type: LogTypeConsume, TokenId: 8, Group: "primary", ModelName: "m", IsStream: true, CreatedAt: 999, UseTime: 1, CompletionTokens: 999, Other: `{"frt":1,"stream_status":{"status":"error"}}`},
{Type: LogTypeConsume, TokenId: 8, Group: "primary", ModelName: "m", IsStream: true, CreatedAt: 999, UseTime: 1, CompletionTokens: 999, Other: `{"frt":1,"stream_status":{"status":"ok","end_reason":"client_gone"}}`},
}
require.NoError(t, db.Create(&noise).Error)
actual, err := GetGroupModelPerformance(ctx, "primary", "m")
require.NoError(t, err)
assert.Equal(t, ModelPerformanceMetric{Value: 16.5, Samples: 30, Source: "calls", UpdatedAt: 130}, *actual.Throughput)
assert.Equal(t, ModelPerformanceMetric{Value: 1650, Samples: 30, Source: "calls", UpdatedAt: 130}, *actual.FirstToken)
// Synchronous requests contribute throughput, not fabricated first-token timing.
require.NoError(t, db.Create(&Log{Type: LogTypeConsume, TokenId: 8, Group: "primary", ModelName: "m", CreatedAt: 1000, CompletionTokens: 10, Other: `{"latency_ms":500,"frt":500}`}).Error)
actual, err = GetGroupModelPerformance(ctx, "primary", "m")
require.NoError(t, err)
assert.InDelta(t, (16.5*30-2+20)/30, actual.Throughput.Value, 0.0001)
assert.Equal(t, 1650.0, actual.FirstToken.Value)
require.NoError(t, db.Model(&Log{}).Where(map[string]any{"group": "primary", "model_name": "m"}).Where("token_id > 0").Update("is_stream", false).Error)
partial, err := GetGroupModelPerformance(ctx, "primary", "m")
require.NoError(t, err)
assert.Equal(t, "calls", partial.Throughput.Source)
assert.Equal(t, "test", partial.FirstToken.Source)
assert.Equal(t, 1000.0, partial.FirstToken.Value)
})
}
}
+43 -40
View File
@@ -2,8 +2,6 @@ package model
import (
"context"
"errors"
"fmt"
"strings"
"time"
@@ -152,21 +150,23 @@ func GetLogByTokenId(tokenId int) (logs []*Log, err error) {
return logs, err
}
func RecordLog(userId int, logType int, content string) {
func RecordLog(userId int, logType int, content *common.Message) {
if logType == LogTypeConsume && !common.LogConsumeEnabled {
return
}
username, _ := GetUsernameById(userId, false)
other := NewLogOther()
log := &Log{
UserId: userId,
Username: username,
CreatedAt: common.GetTimestamp(),
Type: logType,
Content: content,
Content: other.SetContent([]*common.Message{content}),
Other: other.JSONString(),
}
err := createLog(log)
if err != nil {
common.SysLog("failed to record log: " + err.Error())
common.SysLog(common.LogText("failed to record log: %s", err.Error()))
}
}
@@ -202,13 +202,13 @@ func RecordLogWithAdminInfo(userId int, logType int, content string, adminInfo *
if adminInfo != nil || operation != nil {
data, err := common.Marshal(AuditOther{AdminInfo: adminInfo, Op: operation})
if err != nil {
common.SysError("failed to encode log admin info: " + err.Error())
common.SysError(common.LogText("failed to encode log admin info: %s", err.Error()))
return
}
log.Other = string(data)
}
if err := createLog(log); err != nil {
common.SysLog("failed to record log: " + err.Error())
common.SysLog(common.LogText("failed to record log: %s", err.Error()))
}
}
@@ -254,7 +254,7 @@ func RecordOperationAuditLog(logUserId, actorRole int, content string, ip string
RecordAuditLog(c, AuditLog{UserId: logUserId, Username: username, ActorRole: actorRole, Category: category, Action: action, Content: content, Ip: ip, Status: status, Success: success, Other: other})
}
func RecordTopupLog(userId int, content string, callerIp string, paymentMethod string, callbackPaymentMethod string) {
func RecordTopupLog(userId int, content *common.Message, callerIp string, paymentMethod string, callbackPaymentMethod string) {
username, _ := GetUsernameById(userId, false)
other := NewLogOther()
other.MergeAdmin(map[string]any{
@@ -270,19 +270,19 @@ func RecordTopupLog(userId int, content string, callerIp string, paymentMethod s
Username: username,
CreatedAt: common.GetTimestamp(),
Type: LogTypeTopup,
Content: content,
Content: other.SetContent([]*common.Message{content}),
Ip: callerIp,
Other: other.JSONString(),
}
err := createLog(log)
if err != nil {
common.SysLog("failed to record topup log: " + err.Error())
common.SysLog(common.LogText("failed to record topup log: %s", err.Error()))
}
}
func RecordErrorLog(c *gin.Context, userId int, channelId int, modelName string, tokenName string, content string, tokenId int, useTimeSeconds int,
isStream bool, group string, other *LogOther) {
logger.LogInfo(c, fmt.Sprintf("record error log: userId=%d, channelId=%d, modelName=%s, tokenName=%s, content=%s", userId, channelId, modelName, tokenName, common.LocalLogPreview(content)))
logger.LogInfo(c, common.LogText("record error log: userId=%d, channelId=%d, modelName=%s, tokenName=%s, content=%s", userId, channelId, modelName, tokenName, common.LocalLogPreview(content)))
username := c.GetString("username")
requestId := c.GetString(common.RequestIdKey)
upstreamRequestId := c.GetString(common.UpstreamRequestIdKey)
@@ -324,35 +324,37 @@ func RecordErrorLog(c *gin.Context, userId int, channelId int, modelName string,
normalizeErrorLogGroup(log)
err := createLog(log)
if err != nil {
logger.LogError(c, "failed to record log: "+err.Error())
logger.LogError(c, common.LogText("failed to record log: %s", err.Error()))
}
}
type RecordConsumeLogParams struct {
ChannelId int `json:"channel_id"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
ModelName string `json:"model_name"`
TokenName string `json:"token_name"`
Quota int `json:"quota"`
Content string `json:"content"`
TokenId int `json:"token_id"`
UseTimeSeconds int `json:"use_time_seconds"`
IsStream bool `json:"is_stream"`
Group string `json:"group"`
Other *LogOther `json:"other"`
ChannelId int `json:"channel_id"`
PromptTokens int `json:"prompt_tokens"`
CompletionTokens int `json:"completion_tokens"`
ModelName string `json:"model_name"`
TokenName string `json:"token_name"`
Quota int `json:"quota"`
Content []*common.Message `json:"content"`
TokenId int `json:"token_id"`
UseTimeSeconds int `json:"use_time_seconds"`
IsStream bool `json:"is_stream"`
Group string `json:"group"`
Other *LogOther `json:"other"`
}
func RecordConsumeLog(c *gin.Context, userId int, params RecordConsumeLogParams) {
if !common.LogConsumeEnabled {
return
}
logger.LogInfo(c, fmt.Sprintf("record consume log: userId=%d, params=%s", userId, common.GetJsonString(params)))
logger.LogInfo(c, common.LogText("record consume log: userId=%d, params=%s", userId, common.GetJsonString(params)))
username := c.GetString("username")
requestId := c.GetString(common.RequestIdKey)
upstreamRequestId := c.GetString(common.UpstreamRequestIdKey)
createdAt := common.GetTimestamp()
otherStr := params.Other.JSONString()
other := params.Other
content := other.SetContent(params.Content)
otherStr := other.JSONString()
// 判断是否需要记录 IP
needRecordIp := false
if settingMap, err := GetUserSetting(userId, false); err == nil {
@@ -365,7 +367,7 @@ func RecordConsumeLog(c *gin.Context, userId int, params RecordConsumeLogParams)
Username: username,
CreatedAt: createdAt,
Type: LogTypeConsume,
Content: params.Content,
Content: content,
PromptTokens: params.PromptTokens,
CompletionTokens: params.CompletionTokens,
TokenName: params.TokenName,
@@ -389,7 +391,7 @@ func RecordConsumeLog(c *gin.Context, userId int, params RecordConsumeLogParams)
}
err := createLog(log)
if err != nil {
logger.LogError(c, "failed to record log: "+err.Error())
logger.LogError(c, common.LogText("failed to record log: %s", err.Error()))
}
if common.DataExportEnabled {
LogQuotaData(QuotaDataLogParams{
@@ -410,7 +412,7 @@ func RecordConsumeLog(c *gin.Context, userId int, params RecordConsumeLogParams)
type RecordTaskBillingLogParams struct {
UserId int
LogType int
Content string
Content []*common.Message
ChannelId int
ModelName string
Quota int
@@ -432,23 +434,24 @@ func RecordTaskBillingLog(params RecordTaskBillingLogParams) {
}
}
createdAt := common.GetTimestamp()
other := params.Other
log := &Log{
UserId: params.UserId,
Username: username,
CreatedAt: createdAt,
Type: params.LogType,
Content: params.Content,
Content: other.SetContent(params.Content),
TokenName: tokenName,
ModelName: params.ModelName,
Quota: params.Quota,
ChannelId: params.ChannelId,
TokenId: params.TokenId,
Group: params.Group,
Other: params.Other.JSONString(),
Other: other.JSONString(),
}
err := createLog(log)
if err != nil {
common.SysLog("failed to record task billing log: " + err.Error())
common.SysLog(common.LogText("failed to record task billing log: %s", err.Error()))
}
if params.LogType == LogTypeConsume && common.DataExportEnabled {
nodeName := params.NodeName
@@ -606,8 +609,8 @@ func GetUserLogs(userId int, logType int, startTimestamp int64, endTimestamp int
}
err = tx.Model(&Log{}).Limit(logSearchCountLimit).Count(&total).Error
if err != nil {
common.SysError("failed to count user logs: " + err.Error())
return nil, 0, errors.New("查询日志失败")
common.SysError(common.LogText("failed to count user logs: %s", err.Error()))
return nil, 0, common.NewMessage("Failed to query logs")
}
order := "logs.id desc"
if common.UsingLogDatabase(common.DatabaseTypeClickHouse) {
@@ -615,8 +618,8 @@ func GetUserLogs(userId int, logType int, startTimestamp int64, endTimestamp int
}
err = tx.Order(order).Limit(num).Offset(startIdx).Find(&logs).Error
if err != nil {
common.SysError("failed to search user logs: " + err.Error())
return nil, 0, errors.New("查询日志失败")
common.SysError(common.LogText("failed to search user logs: %s", err.Error()))
return nil, 0, common.NewMessage("Failed to query logs")
}
formatUserLogs(logs, startIdx)
@@ -683,16 +686,16 @@ func SumUsedQuota(logType int, startTimestamp int64, endTimestamp int64, modelNa
// 执行查询
if err := tx.Scan(&stat).Error; err != nil {
common.SysError("failed to query log stat: " + err.Error())
return stat, errors.New("查询统计数据失败")
common.SysError(common.LogText("failed to query log stat: %s", err.Error()))
return stat, common.NewMessage("Failed to query statistics")
}
var rateStat struct {
Rpm int
Tpm int
}
if err := rpmTpmQuery.Scan(&rateStat).Error; err != nil {
common.SysError("failed to query rpm/tpm stat: " + err.Error())
return stat, errors.New("查询统计数据失败")
common.SysError(common.LogText("failed to query rpm/tpm stat: %s", err.Error()))
return stat, common.NewMessage("Failed to query statistics")
}
stat.Rpm = rateStat.Rpm
stat.Tpm = rateStat.Tpm
+21 -4
View File
@@ -4,14 +4,16 @@ import (
"encoding/json"
"maps"
"slices"
"strings"
"github.com/QuantumNous/new-api/common"
)
const (
logOtherAdminInfoKey = "admin_info"
logOtherRootInfoKey = "root_info"
logOtherAuditInfoKey = "audit_info"
logOtherAdminInfoKey = "admin_info"
logOtherRootInfoKey = "root_info"
logOtherAuditInfoKey = "audit_info"
logOtherContentPartsKey = "content_parts"
)
// legacySensitiveLogOtherKeys are historical top-level fields that must never
@@ -74,6 +76,21 @@ func (o *LogOther) MergePublic(values map[string]any) {
}
}
// SetContent stores structured content under content_parts, which the web
// console renders in the viewer's language, and returns the English text kept
// in the content column for exports and API callers.
func (o *LogOther) SetContent(parts []*common.Message) string {
if len(parts) == 0 {
return ""
}
o.SetPublic(logOtherContentPartsKey, parts)
texts := make([]string, 0, len(parts))
for _, part := range parts {
texts = append(texts, part.Error())
}
return strings.Join(texts, ", ")
}
func (o *LogOther) SetAdmin(key string, value any) bool {
if o == nil || key == "" {
return false
@@ -159,7 +176,7 @@ func (o *LogOther) jsonString() string {
}
data, err := common.Marshal(o.toMap())
if err != nil {
common.SysError("failed to marshal log other: " + err.Error())
common.SysError(common.LogText("failed to marshal log other: %s", err.Error()))
return ""
}
return string(data)
+8 -8
View File
@@ -13,26 +13,26 @@ func TestErrorLogGroupProjection(t *testing.T) {
{"root", FormatRootLogs}, {"admin", FormatAdminLogs}, {"user", func(logs []*Log) { formatUserLogs(logs, 0) }},
} {
t.Run(format.name, func(t *testing.T) {
metadata := `{"admin_info":{"request_policy":[{"channel_id":101,"group":"example-primary"},{"channel_id":102,"group":"example-preferred"}],"private":"secret"},"root_info":{"private":"secret"}}`
metadata := `{"admin_info":{"request_policy":[{"channel_id":1,"group":"test-group-a"},{"channel_id":27,"group":"test-group-b"}],"private":"secret"},"root_info":{"private":"secret"}}`
logs := []*Log{
{Type: LogTypeError, ChannelId: 102, Group: "default", Other: metadata},
{Type: LogTypeError, ChannelId: 101, Group: "default", Other: metadata},
{Type: LogTypeError, ChannelId: 27, Group: "default", Other: metadata},
{Type: LogTypeError, ChannelId: 1, Group: "default", Other: metadata},
{Type: LogTypeError, ChannelId: 0, Group: "default", Other: metadata},
{Type: LogTypeConsume, ChannelId: 102, Group: "billed-group", Quota: 12345, Other: metadata},
{Type: LogTypeConsume, ChannelId: 27, Group: "billed-group", Quota: 799463, Other: metadata},
{Type: LogTypeError, ChannelId: 99, Group: "legacy", Other: metadata},
{Type: LogTypeError, ChannelId: 102, Group: "legacy", Other: `invalid`},
{Type: LogTypeError, ChannelId: 27, Group: "legacy", Other: `invalid`},
}
format.fn(logs)
for i, want := range []string{"example-preferred", "example-primary", "", "billed-group", "legacy", "legacy"} {
for i, want := range []string{"test-group-b", "test-group-a", "", "billed-group", "legacy", "legacy"} {
require.Equal(t, want, logs[i].Group)
}
require.Equal(t, 12345, logs[3].Quota)
require.Equal(t, 799463, logs[3].Quota)
if format.name == "user" {
require.NotContains(t, logs[0].Other, "secret")
require.NotContains(t, logs[0].Other, "request_policy")
}
format.fn(logs)
require.Equal(t, "example-preferred", logs[0].Group)
require.Equal(t, "test-group-b", logs[0].Group)
})
}
}
+62 -34
View File
@@ -1,6 +1,7 @@
package model
import (
"database/sql"
"fmt"
"log"
"net/url"
@@ -71,7 +72,7 @@ func createRootAccountIfNeed() error {
var user User
//if user.Status != common.UserStatusEnabled {
if err := DB.First(&user).Error; err != nil {
common.SysLog("no user exists, create a root user for you: username is root, password is 123456")
common.SysLog(common.LogText("no user exists, create a root user for you: username is root, password is 123456"))
hashedPassword, err := common.Password2Hash("123456")
if err != nil {
return err
@@ -95,7 +96,7 @@ func CheckSetup() {
if setup == nil {
// No setup record exists, check if we have a root user
if RootUserExists() {
common.SysLog("system is not initialized, but root user exists")
common.SysLog(common.LogText("system is not initialized, but root user exists"))
// Create setup record
newSetup := Setup{
Version: common.Version,
@@ -103,16 +104,16 @@ func CheckSetup() {
}
err := DB.Create(&newSetup).Error
if err != nil {
common.SysLog("failed to create setup record: " + err.Error())
common.SysLog(common.LogText("failed to create setup record: %s", err.Error()))
}
constant.Setup = true
} else {
common.SysLog("system is not initialized and no root user exists")
common.SysLog(common.LogText("system is not initialized and no root user exists"))
constant.Setup = false
}
} else {
// Setup record exists, system is initialized
common.SysLog("system is already initialized at: " + time.Unix(setup.InitializedAt, 0).String())
common.SysLog(common.LogText("system is already initialized at: %s", time.Unix(setup.InitializedAt, 0).String()))
constant.Setup = true
}
}
@@ -144,13 +145,13 @@ func chooseDB(envName string, isLog bool) (*gorm.DB, common.DatabaseType, error)
if !isLog {
return nil, "", fmt.Errorf("%s does not support ClickHouse; use SQLite, MySQL, or PostgreSQL for the primary database and LOG_SQL_DSN for ClickHouse logs", envName)
}
common.SysLog("using ClickHouse as log database")
common.SysLog(common.LogText("using ClickHouse as log database"))
db, err := gorm.Open(clickhouse.Open(normalizeClickHouseDSN(dsn)), newGormConfig(false))
return db, common.DatabaseTypeClickHouse, err
}
if strings.HasPrefix(dsn, "postgres://") || strings.HasPrefix(dsn, "postgresql://") {
// Use PostgreSQL
common.SysLog("using PostgreSQL as database")
common.SysLog(common.LogText("using PostgreSQL as database"))
// 同时关闭 pgx 隐式与 GORM 显式预处理语句:命名 prepared statement 与
// 事务池代理(PgBouncer/Neon/Supabase)不兼容,会触发 FATAL 08P01/42P05。
db, err := gorm.Open(postgresMigrationDialector{postgres.Dialector{Config: &postgres.Config{
@@ -160,12 +161,12 @@ func chooseDB(envName string, isLog bool) (*gorm.DB, common.DatabaseType, error)
return db, common.DatabaseTypePostgreSQL, err
}
if strings.HasPrefix(dsn, "local") {
common.SysLog("SQL_DSN not set, using SQLite as database")
common.SysLog(common.LogText("SQL_DSN not set, using SQLite as database"))
db, err := gorm.Open(sqlite.Open(common.SQLitePath), newGormConfig(true))
return db, common.DatabaseTypeSQLite, err
}
// Use MySQL
common.SysLog("using MySQL as database")
common.SysLog(common.LogText("using MySQL as database"))
// check parseTime
if !strings.Contains(dsn, "parseTime") {
if strings.Contains(dsn, "?") {
@@ -178,7 +179,7 @@ func chooseDB(envName string, isLog bool) (*gorm.DB, common.DatabaseType, error)
return db, common.DatabaseTypeMySQL, err
}
// Use SQLite
common.SysLog("SQL_DSN not set, using SQLite as database")
common.SysLog(common.LogText("SQL_DSN not set, using SQLite as database"))
db, err := gorm.Open(sqlite.Open(common.SQLitePath), newGormConfig(true))
return db, common.DatabaseTypeSQLite, err
}
@@ -208,17 +209,23 @@ func InitDB() (err error) {
if err != nil {
return err
}
sqlDB.SetMaxIdleConns(common.GetEnvOrDefault("SQL_MAX_IDLE_CONNS", 100))
sqlDB.SetMaxOpenConns(common.GetEnvOrDefault("SQL_MAX_OPEN_CONNS", 1000))
sqlDB.SetMaxIdleConns(common.GetEnvOrDefault("SQL_MAX_IDLE_CONNS", 10))
sqlDB.SetMaxOpenConns(common.GetEnvOrDefault("SQL_MAX_OPEN_CONNS", 40))
sqlDB.SetConnMaxLifetime(time.Second * time.Duration(common.GetEnvOrDefault("SQL_MAX_LIFETIME", 60)))
monitorDatabasePool("main", sqlDB)
if !common.IsMasterNode {
// Only the master node migrates. A node that cannot read the deadline
// keeps rejecting legacy access tokens instead of refusing to start.
if err := EnsureLegacyAccessTokenRetireAt(common.GetTimestamp()); err != nil {
common.SysError(common.LogText("initialize legacy access token deadline: %s", err.Error()))
}
return nil
}
if common.UsingMainDatabase(common.DatabaseTypeMySQL) {
//_, _ = sqlDB.Exec("ALTER TABLE channels MODIFY model_mapping TEXT;") // TODO: delete this line when most users have upgraded
}
common.SysLog("database migration started")
common.SysLog(common.LogText("database migration started"))
err = migrateDB()
if err == nil {
err = MigrateTokenAstraAutoSwitch()
@@ -268,14 +275,15 @@ func InitLogDB() (err error) {
if err != nil {
return err
}
sqlDB.SetMaxIdleConns(common.GetEnvOrDefault("SQL_MAX_IDLE_CONNS", 100))
sqlDB.SetMaxOpenConns(common.GetEnvOrDefault("SQL_MAX_OPEN_CONNS", 1000))
sqlDB.SetMaxIdleConns(common.GetEnvOrDefault("SQL_MAX_IDLE_CONNS", 3))
sqlDB.SetMaxOpenConns(common.GetEnvOrDefault("SQL_MAX_OPEN_CONNS", 10))
sqlDB.SetConnMaxLifetime(time.Second * time.Duration(common.GetEnvOrDefault("SQL_MAX_LIFETIME", 60)))
monitorDatabasePool("log", sqlDB)
if !common.IsMasterNode {
return nil
}
common.SysLog("database migration started")
common.SysLog(common.LogText("database migration started"))
err = migrateLOGDB()
return err
} else {
@@ -284,6 +292,22 @@ func InitLogDB() (err error) {
return err
}
func monitorDatabasePool(name string, db *sql.DB) {
go func() {
var lastWaitCount int64
ticker := time.NewTicker(30 * time.Second)
defer ticker.Stop()
for range ticker.C {
stats := db.Stats()
if stats.WaitCount == lastWaitCount && stats.InUse < stats.MaxOpenConnections {
continue
}
lastWaitCount = stats.WaitCount
common.SysError(common.LogText("database pool %s: open=%d in_use=%d idle=%d max_open=%d wait_count=%d wait_duration=%s", name, stats.OpenConnections, stats.InUse, stats.Idle, stats.MaxOpenConnections, stats.WaitCount, stats.WaitDuration))
}
}()
}
var userQuotaColumns = []string{"quota", "used_quota", "aff_quota", "aff_history"}
// ensureUserQuotaColumns rejects a legacy 32-bit wallet schema before any
@@ -291,7 +315,7 @@ var userQuotaColumns = []string{"quota", "used_quota", "aff_quota", "aff_history
// an existing wallet; operators must migrate it explicitly before starting.
func ensureUserQuotaColumns(db *gorm.DB, dbType common.DatabaseType) error {
if common.GetEnvOrDefaultBool("SKIP_64BIT_QUOTA_SCHEMA_CHECK", false) {
common.SysLog("SKIP_64BIT_QUOTA_SCHEMA_CHECK=true; skipping user quota schema check")
common.SysLog(common.LogText("SKIP_64BIT_QUOTA_SCHEMA_CHECK=true; skipping user quota schema check"))
return nil
}
if db == nil || dbType == common.DatabaseTypeSQLite {
@@ -344,7 +368,7 @@ func migrateDB() error {
return err
}
if err := migrateOptionPrimaryKey(DB); err != nil {
common.SysError("failed to migrate options primary key: " + err.Error())
common.SysError(common.LogText("failed to migrate options primary key: %s", err.Error()))
}
err := DB.AutoMigrate(
@@ -389,6 +413,7 @@ func migrateDB() error {
&DetectionSchedule{},
&CasbinRule{},
&AuthzRole{},
&UserAccessToken{},
)
if err != nil {
return err
@@ -396,6 +421,9 @@ func migrateDB() error {
if err := InitializeUserAuthVersions(); err != nil {
return err
}
if err := EnsureLegacyAccessTokenRetireAt(common.GetTimestamp()); err != nil {
return fmt.Errorf("initialize legacy access token deadline: %w", err)
}
if err := InitializeExternalIdentityClaims(); err != nil {
return err
}
@@ -619,7 +647,7 @@ func migrateTokenModelLimitsToText() error {
if err := DB.Raw(`SELECT data_type FROM information_schema.columns
WHERE table_schema = current_schema() AND table_name = ? AND column_name = ?`,
tableName, columnName).Scan(&dataType).Error; err != nil {
common.SysLog(fmt.Sprintf("Warning: failed to query metadata for %s.%s: %v", tableName, columnName, err))
common.SysLog(common.LogText("Warning: failed to query metadata for %s.%s: %v", tableName, columnName, err))
} else if dataType == "text" {
return nil
}
@@ -629,7 +657,7 @@ func migrateTokenModelLimitsToText() error {
if err := DB.Raw(`SELECT COLUMN_TYPE FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = ? AND column_name = ?`,
tableName, columnName).Scan(&columnType).Error; err != nil {
common.SysLog(fmt.Sprintf("Warning: failed to query metadata for %s.%s: %v", tableName, columnName, err))
common.SysLog(common.LogText("Warning: failed to query metadata for %s.%s: %v", tableName, columnName, err))
} else if strings.ToLower(columnType) == "text" {
return nil
}
@@ -642,7 +670,7 @@ func migrateTokenModelLimitsToText() error {
if err := DB.Exec(alterSQL).Error; err != nil {
return fmt.Errorf("failed to migrate %s.%s to text: %w", tableName, columnName, err)
}
common.SysLog(fmt.Sprintf("Successfully migrated %s.%s to text", tableName, columnName))
common.SysLog(common.LogText("Successfully migrated %s.%s to text", tableName, columnName))
}
return nil
}
@@ -676,7 +704,7 @@ func migrateSubscriptionPlanPriceAmount() {
if err := DB.Raw(`SELECT data_type FROM information_schema.columns
WHERE table_schema = current_schema() AND table_name = ? AND column_name = ?`,
tableName, columnName).Scan(&dataType).Error; err != nil {
common.SysLog(fmt.Sprintf("Warning: failed to query metadata for %s.%s: %v", tableName, columnName, err))
common.SysLog(common.LogText("Warning: failed to query metadata for %s.%s: %v", tableName, columnName, err))
} else if dataType == "numeric" {
return // Already decimal/numeric
}
@@ -688,7 +716,7 @@ func migrateSubscriptionPlanPriceAmount() {
if err := DB.Raw(`SELECT COLUMN_TYPE FROM information_schema.columns
WHERE table_schema = DATABASE() AND table_name = ? AND column_name = ?`,
tableName, columnName).Scan(&columnType).Error; err != nil {
common.SysLog(fmt.Sprintf("Warning: failed to query metadata for %s.%s: %v", tableName, columnName, err))
common.SysLog(common.LogText("Warning: failed to query metadata for %s.%s: %v", tableName, columnName, err))
} else if strings.HasPrefix(strings.ToLower(columnType), "decimal") {
return // Already decimal
}
@@ -700,9 +728,9 @@ func migrateSubscriptionPlanPriceAmount() {
if alterSQL != "" {
if err := DB.Exec(alterSQL).Error; err != nil {
common.SysLog(fmt.Sprintf("Warning: failed to migrate %s.%s to decimal: %v", tableName, columnName, err))
common.SysLog(common.LogText("Warning: failed to migrate %s.%s to decimal: %v", tableName, columnName, err))
} else {
common.SysLog(fmt.Sprintf("Successfully migrated %s.%s to decimal(10,6)", tableName, columnName))
common.SysLog(common.LogText("Successfully migrated %s.%s to decimal(10,6)", tableName, columnName))
}
}
}
@@ -736,7 +764,7 @@ func checkMySQLChineseSupport(db *gorm.DB) error {
var schemaCharset, schemaCollation string
err := db.Raw("SELECT DEFAULT_CHARACTER_SET_NAME, DEFAULT_COLLATION_NAME FROM information_schema.SCHEMATA WHERE SCHEMA_NAME = DATABASE()").Row().Scan(&schemaCharset, &schemaCollation)
if err != nil {
return fmt.Errorf("读取当前库默认字符集/排序规则失败 / Failed to read schema default charset/collation: %v", err)
return fmt.Errorf("Failed to read schema default charset/collation: %v", err)
}
toLower := func(s string) string { return strings.ToLower(s) }
@@ -768,8 +796,8 @@ func checkMySQLChineseSupport(db *gorm.DB) error {
// 1) 当前库默认值必须支持中文
if !isChineseCapable(schemaCharset, schemaCollation) {
return fmt.Errorf("当前库默认字符集/排序规则不支持中文:schema(%s/%s)。请将库设置为 utf8mb4/utf8/gbk/big5/gb18030 / Schema default charset/collation is not Chinese-capable: schema(%s/%s). Please set to utf8mb4/utf8/gbk/big5/gb18030",
schemaCharset, schemaCollation, schemaCharset, schemaCollation)
return fmt.Errorf("Schema default charset/collation is not Chinese-capable: schema(%s/%s). Please set to utf8mb4/utf8/gbk/big5/gb18030",
schemaCharset, schemaCollation)
}
// 2) 所有物理表的排序规则(隐含字符集)必须支持中文
@@ -779,7 +807,7 @@ func checkMySQLChineseSupport(db *gorm.DB) error {
}
var tables []tableInfo
if err := db.Raw("SELECT TABLE_NAME, TABLE_COLLATION FROM information_schema.TABLES WHERE TABLE_SCHEMA = DATABASE() AND TABLE_TYPE = 'BASE TABLE'").Scan(&tables).Error; err != nil {
return fmt.Errorf("读取表排序规则失败 / Failed to read table collations: %v", err)
return fmt.Errorf("Failed to read table collations: %v", err)
}
var badTables []string
@@ -811,8 +839,8 @@ func checkMySQLChineseSupport(db *gorm.DB) error {
shown = shown[:maxShow]
}
return fmt.Errorf(
"存在不支持中文的表,请修复其排序规则/字符集。示例(最多展示 %d 项):%v / Found tables not Chinese-capable. Please fix their collation/charset. Examples (showing up to %d): %v",
maxShow, shown, maxShow, shown,
"Found tables not Chinese-capable. Please fix their collation/charset. Examples (showing up to %d): %v",
maxShow, shown,
)
}
return nil
@@ -833,17 +861,17 @@ func PingDB() error {
sqlDB, err := DB.DB()
if err != nil {
log.Printf("Error getting sql.DB from GORM: %v", err)
log.Print(common.LogText("Error getting sql.DB from GORM: %v", err))
return err
}
err = sqlDB.Ping()
if err != nil {
log.Printf("Error pinging DB: %v", err)
log.Print(common.LogText("Error pinging DB: %v", err))
return err
}
lastPingTime = time.Now()
common.SysLog("Database pinged successfully")
common.SysLog(common.LogText("Database pinged successfully"))
return nil
}
+22 -9
View File
@@ -379,10 +379,15 @@ func validateModelPricing(name string, values, previous PricingValues) error {
return fmt.Errorf("model %s: plugin %s: billing expression is required", name, key)
}
plugin, exists := generation.Get(key)
if !exists || !slices.Contains(plugin.Meta.Models, name) {
if previousVariants[key] == expression {
continue
}
declared := exists && slices.Contains(plugin.Meta.Models, name)
// An unchanged override stays saveable after its plugin disappears or
// stops declaring the model. A sole provider also keeps it after
// narrowing its usage profile; relay checks shared models against
// each profile, so their prices must follow it.
if previousVariants[key] == expression && (!declared || !generation.SharedModel(name)) {
continue
}
if !declared {
return fmt.Errorf("model %s: plugin %s does not declare this model", name, key)
}
schema, _ := plugin.Meta.UsageForModel(name)
@@ -414,10 +419,16 @@ func validateModelPricing(name string, values, previous PricingValues) error {
if _, err := billingexpr.CompileFromCache(expression); err != nil {
return fmt.Errorf("model %s: %w", name, err)
}
// An unchanged expression a sole provider has been charging with keeps
// its price after the plugin narrows its usage profile; relay checks
// shared models against each profile, so their prices must follow it.
unchanged := previous[key] == expression
var err error
if plugins := generation.PluginsByModel(name); len(plugins) > 0 {
for _, plugin := range plugins {
if _, overridden := variants[plugin.Meta.Key]; overridden {
_, overridden := variants[plugin.Meta.Key]
_, wasOverridden := previousVariants[plugin.Meta.Key]
if overridden || (unchanged && !wasOverridden && len(plugins) == 1) {
continue
}
schema, _ := plugin.Meta.UsageForModel(name)
@@ -427,12 +438,14 @@ func validateModelPricing(name string, values, previous PricingValues) error {
}
} else if target, resolved := ResolveTaskModelAlias(generation, name); resolved {
if plugin, ok := generation.Get(target.PluginKey); ok {
schema, _ := plugin.Meta.UsageForModel(target.Declared)
err = billing_setting.SmokeTestTaskExpr(expression, schema)
if !unchanged || generation.SharedModel(target.Declared) {
schema, _ := plugin.Meta.UsageForModel(target.Declared)
err = billing_setting.SmokeTestTaskExpr(expression, schema)
}
} else {
err = billing_setting.SmokeTestExpr(expression)
}
} else if previous[key] != expression || len(billingexpr.UsedUsageKeys(expression)) == 0 {
} else if !unchanged || len(billingexpr.UsedUsageKeys(expression)) == 0 {
err = billing_setting.SmokeTestExpr(expression)
}
// With no remaining plugin, an unchanged stored usage expression has
@@ -542,7 +555,7 @@ func mutateModelPricingOptions(mutate func(*gorm.DB, map[string]map[string]any)
return err
}
if len(duplicated) > 0 {
common.SysError("options table has duplicate pricing keys [" + strings.Join(duplicated, ", ") + "]; the table is missing a primary key")
common.SysError(common.LogText("options table has duplicate pricing keys [%s]; the table is missing a primary key", strings.Join(duplicated, ", ")))
}
defaults := defaultPricingMaps()
for _, key := range modelPricingOptionKeys {
+48 -17
View File
@@ -141,6 +141,51 @@ func ResolveCacheWriteMode(name string, configured PricingValues) CacheWriteMode
return CacheWriteNone
}
// declaredEndpointNames resolves the endpoint types declared by one model
// metadata row. Model metadata accepts two shapes — the map form (custom paths)
// and the type-array form (declared protocols) — and conversion must honour
// both. Reading only the map form made array-form metadata look like a broken
// routing configuration and blocked an otherwise convertible model.
func declaredEndpointNames(raw string) ([]string, error) {
if strings.TrimSpace(raw) == "" {
return nil, nil
}
var value any
if err := common.UnmarshalJsonStr(raw, &value); err != nil {
return nil, err
}
// Normalize whitespace here: the callers match these names against the
// endpoint switch, so a padded value such as "image-generation " would
// silently fall through and drop the image-count settlement rule.
switch endpoints := value.(type) {
case []any:
names := make([]string, 0, len(endpoints))
for _, endpoint := range endpoints {
text, ok := endpoint.(string)
if !ok {
return nil, errors.New("invalid endpoint type")
}
text = strings.TrimSpace(text)
if text == "" {
return nil, errors.New("invalid endpoint type")
}
names = append(names, text)
}
return names, nil
case map[string]any:
names := make([]string, 0, len(endpoints))
for endpoint := range endpoints {
endpoint = strings.TrimSpace(endpoint)
if endpoint == "" {
continue
}
names = append(names, endpoint)
}
return names, nil
}
return nil, errors.New("invalid endpoints")
}
func PreviewModelPricingConversion(name string, draft PricingValues) (*ModelPricingConversion, error) {
if draft == nil {
return nil, errors.New("pricing draft is required")
@@ -188,7 +233,6 @@ func PreviewModelPricingConversion(name string, draft PricingValues) (*ModelPric
return nil, err
}
names := []string{name}
aliPromptExtend, otherImageRoute := false, false
for _, channel := range channels {
if !slices.Contains(channel.GetModels(), name) {
continue
@@ -230,13 +274,6 @@ func PreviewModelPricingConversion(name string, draft PricingValues) (*ModelPric
names = append(names, upstream)
}
}
if channel.Status == common.ChannelStatusEnabled {
if channel.Type == constant.ChannelTypeAli && strings.Contains(upstream, "z-image") {
aliPromptExtend = true
} else {
otherImageRoute = true
}
}
}
for _, modelName := range names {
lower := strings.ToLower(modelName)
@@ -257,11 +294,11 @@ func PreviewModelPricingConversion(name string, draft PricingValues) (*ModelPric
if entry.Endpoints == "" || !slices.ContainsFunc(names, entry.MatchesName) {
continue
}
var endpoints map[string]any
if err := common.UnmarshalJsonStr(entry.Endpoints, &endpoints); err != nil {
endpoints, err := declaredEndpointNames(entry.Endpoints)
if err != nil {
return &ModelPricingConversion{UnsupportedReason: "The model routing configuration could not be verified."}, nil
}
for endpoint := range endpoints {
for _, endpoint := range endpoints {
switch constant.EndpointType(endpoint) {
case constant.EndpointTypeImageGeneration:
if fixedPrice {
@@ -279,13 +316,7 @@ func PreviewModelPricingConversion(name string, draft PricingValues) (*ModelPric
price := effective["ModelPrice"].(float64)
expression = `tier("request", fixed(` + decimal.NewFromFloat(price).String() + `))`
if preview.BillingDetails.ImageCount {
if aliPromptExtend && otherImageRoute {
return &ModelPricingConversion{UnsupportedReason: "This model has different image request multipliers across channels. Use separate billing model names to convert them."}, nil
}
expression = `tier("image", fixed(` + decimal.NewFromFloat(price).String() + `)) * image_count`
if aliPromptExtend {
preview.BillingDetails.RequestRules = append(preview.BillingDetails.RequestRules, LegacyPricingRule{`param("parameters.prompt_extend") == true`, common.ZImagePromptExtendMultiplier})
}
for _, rule := range preview.BillingDetails.RequestRules {
expression += ` * (` + rule.Condition + ` ? ` + decimal.NewFromFloat(rule.Multiplier).String() + ` : 1)`
}
+16 -4
View File
@@ -143,6 +143,7 @@ func InitOptionMap() {
common.OptionMap["QuotaForNewUser"] = strconv.Itoa(common.QuotaForNewUser)
common.OptionMap["QuotaForInviter"] = strconv.Itoa(common.QuotaForInviter)
common.OptionMap["QuotaForInvitee"] = strconv.Itoa(common.QuotaForInvitee)
common.OptionMap["QuotaRemindEnabled"] = strconv.FormatBool(common.QuotaRemindEnabled)
common.OptionMap["QuotaRemindThreshold"] = strconv.Itoa(common.QuotaRemindThreshold)
common.OptionMap["PreConsumedQuota"] = strconv.Itoa(common.PreConsumedQuota)
common.OptionMap["ModelRequestRateLimitCount"] = strconv.Itoa(setting.ModelRequestRateLimitCount)
@@ -199,7 +200,7 @@ func loadOptionsFromDatabase() {
defer requestPolicyOptionMutex.Unlock()
defer func() {
if err := refreshRequestPolicySnapshot(); err != nil {
common.SysError("invalid request policy: " + err.Error())
common.SysError(common.LogText("invalid request policy: %s", err.Error()))
}
}()
passkeyOptionMutex.Lock()
@@ -213,7 +214,7 @@ func loadOptionsFromDatabase() {
}
err := updateOptionMap(option.Key, option.Value)
if err != nil {
common.SysLog("failed to update option map: " + err.Error())
common.SysLog(common.LogText("failed to update option map: %s", err.Error()))
}
}
applyPasskeyDomainOptions(passkeyOptions)
@@ -222,18 +223,27 @@ func loadOptionsFromDatabase() {
func SyncOptions(frequency int) {
for {
time.Sleep(time.Duration(frequency) * time.Second)
common.SysLog("syncing options from database")
common.SysLog(common.LogText("syncing options from database"))
loadOptionsFromDatabase()
}
}
func validateOptionValue(key string, value string) error {
if key == legacyAccessTokenRetireAtKey {
return errLegacyRetireAtReadOnly
}
if err := operation_setting.ValidateQuotaOption(key, value); err != nil {
return err
}
if key == operation_setting.ToolPriceOptionKey {
return operation_setting.ValidateToolPricesJSON(value)
}
if key == operation_setting.ChannelTestConcurrencyOptionKey {
return operation_setting.ValidateChannelTestConcurrency(value)
}
if key == operation_setting.LogMaxRangeDaysOptionKey {
return operation_setting.ValidateLogMaxRangeDays(value)
}
if key == "MaxTokenAutoGroups" {
return setting.ValidateMaxTokenAutoGroups(value)
}
@@ -337,7 +347,7 @@ func UpdateOptionsBulk(values map[string]string) error {
}
func updateOptionMap(key string, value string) (err error) {
if key == retiredThemeOptionKey {
if key == retiredThemeOptionKey || key == legacyAccessTokenRetireAtKey {
common.OptionMapRWMutex.Lock()
delete(common.OptionMap, key)
common.OptionMapRWMutex.Unlock()
@@ -395,6 +405,8 @@ func updateOptionMap(key string, value string) (err error) {
common.AutomaticDisableChannelEnabled = boolValue
case "AutomaticEnableChannelEnabled":
common.AutomaticEnableChannelEnabled = boolValue
case "QuotaRemindEnabled":
common.QuotaRemindEnabled = boolValue
case "LogConsumeEnabled":
common.LogConsumeEnabled = boolValue
case "DisplayInCurrencyEnabled":
+3 -3
View File
@@ -159,7 +159,7 @@ func repairOptionPrimaryKey(db *gorm.DB) error {
if !unique {
return fmt.Errorf("options table still has no unique key after rebuild")
}
common.SysLog(fmt.Sprintf("rebuilt options table with primary key from %d rows into %d keys; previous rows kept in %s", len(rows)-skippedEmpty, len(deduped), backup))
common.SysLog(common.LogText("rebuilt options table with primary key from %d rows into %d keys; previous rows kept in %s", len(rows)-skippedEmpty, len(deduped), backup))
return nil
}
@@ -174,7 +174,7 @@ func dedupeOptionRows(rows []Option) ([]Option, int) {
}
if i, ok := seen[row.Key]; ok {
if deduped[i].Value != row.Value {
common.SysError(fmt.Sprintf("options key %q had conflicting values; keeping the last row", row.Key))
common.SysError(common.LogText("options key %q had conflicting values; keeping the last row", row.Key))
}
deduped[i].Value = row.Value
continue
@@ -183,7 +183,7 @@ func dedupeOptionRows(rows []Option) ([]Option, int) {
deduped = append(deduped, Option{Key: row.Key, Value: row.Value})
}
if skippedEmpty > 0 {
common.SysError(fmt.Sprintf("skipped %d options rows with empty keys while rebuilding the primary key", skippedEmpty))
common.SysError(common.LogText("skipped %d options rows with empty keys while rebuilding the primary key", skippedEmpty))
}
return deduped, skippedEmpty
}
+13 -14
View File
@@ -3,7 +3,6 @@ package model
import (
"encoding/base64"
"errors"
"fmt"
"strings"
"time"
@@ -17,7 +16,7 @@ import (
var (
ErrPasskeyNotFound = errors.New("passkey credential not found")
ErrFriendlyPasskeyNotFound = errors.New("Passkey 验证失败,请重试或联系管理员")
ErrFriendlyPasskeyNotFound = errors.New("Passkey verification failed. Please try again or contact the administrator.")
)
type PasskeyCredential struct {
@@ -124,7 +123,7 @@ func NewPasskeyCredentialFromWebAuthn(userID int, credential *webauthn.Credentia
func GetPasskeyByUserID(userID int) (*PasskeyCredential, error) {
if userID == 0 {
common.SysLog("GetPasskeyByUserID: empty user ID")
common.SysLog(common.LogText("GetPasskeyByUserID: empty user ID"))
return nil, ErrFriendlyPasskeyNotFound
}
var credential PasskeyCredential
@@ -134,7 +133,7 @@ func GetPasskeyByUserID(userID int) (*PasskeyCredential, error) {
return nil, ErrPasskeyNotFound
}
// 只有真正的数据库错误才记录日志
common.SysLog(fmt.Sprintf("GetPasskeyByUserID: database error for user %d: %v", userID, err))
common.SysLog(common.LogText("GetPasskeyByUserID: database error for user %d: %v", userID, err))
return nil, ErrFriendlyPasskeyNotFound
}
return &credential, nil
@@ -142,7 +141,7 @@ func GetPasskeyByUserID(userID int) (*PasskeyCredential, error) {
func GetPasskeyByCredentialID(credentialID []byte) (*PasskeyCredential, error) {
if len(credentialID) == 0 {
common.SysLog("GetPasskeyByCredentialID: empty credential ID")
common.SysLog(common.LogText("GetPasskeyByCredentialID: empty credential ID"))
return nil, ErrFriendlyPasskeyNotFound
}
@@ -150,10 +149,10 @@ func GetPasskeyByCredentialID(credentialID []byte) (*PasskeyCredential, error) {
var credential PasskeyCredential
if err := DB.Where("credential_id = ?", credIDStr).First(&credential).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
common.SysLog(fmt.Sprintf("GetPasskeyByCredentialID: passkey not found for credential ID length %d", len(credentialID)))
common.SysLog(common.LogText("GetPasskeyByCredentialID: passkey not found for credential ID length %d", len(credentialID)))
return nil, ErrFriendlyPasskeyNotFound
}
common.SysLog(fmt.Sprintf("GetPasskeyByCredentialID: database error for credential ID: %v", err))
common.SysLog(common.LogText("GetPasskeyByCredentialID: database error for credential ID: %v", err))
return nil, ErrFriendlyPasskeyNotFound
}
@@ -165,7 +164,7 @@ func GetPasskeyByCredentialID(credentialID []byte) (*PasskeyCredential, error) {
// transports and attestation metadata) is immutable on this path.
func UpdatePasskeyAssertionState(userID int, credential *webauthn.Credential, lastUsedAt time.Time, rpID string) error {
if userID <= 0 || credential == nil || len(credential.ID) == 0 || lastUsedAt.IsZero() || rpID == "" {
return fmt.Errorf("Passkey 保存失败,请重试")
return errors.New("failed to save the Passkey, please try again")
}
credentialID := base64.StdEncoding.EncodeToString(credential.ID)
passkeyOptionMutex.Lock()
@@ -210,12 +209,12 @@ func UpdatePasskeyAssertionState(userID int, credential *webauthn.Credential, la
func upsertPasskeyCredentialWithTx(tx *gorm.DB, credential *PasskeyCredential) error {
if err := tx.Unscoped().Where("user_id = ?", credential.UserID).Delete(&PasskeyCredential{}).Error; err != nil {
common.SysLog(fmt.Sprintf("UpsertPasskeyCredential: failed to delete existing credential for user %d: %v", credential.UserID, err))
return fmt.Errorf("Passkey 保存失败,请重试")
common.SysLog(common.LogText("UpsertPasskeyCredential: failed to delete existing credential for user %d: %v", credential.UserID, err))
return errors.New("failed to save the Passkey, please try again")
}
if err := tx.Create(credential).Error; err != nil {
common.SysLog(fmt.Sprintf("UpsertPasskeyCredential: failed to create credential for user %d: %v", credential.UserID, err))
return fmt.Errorf("Passkey 保存失败,请重试")
common.SysLog(common.LogText("UpsertPasskeyCredential: failed to create credential for user %d: %v", credential.UserID, err))
return errors.New("failed to save the Passkey, please try again")
}
return nil
}
@@ -232,7 +231,7 @@ func RegisterPasskeyForSession(identity AuthSessionIdentity, credential *Passkey
func upsertPasskeyCredentialWithAuthVersion(credential *PasskeyCredential, identity *AuthSessionIdentity) error {
if credential == nil || credential.UserID <= 0 {
return fmt.Errorf("Passkey 保存失败,请重试")
return errors.New("failed to save the Passkey, please try again")
}
passkeyOptionMutex.Lock()
defer passkeyOptionMutex.Unlock()
@@ -273,7 +272,7 @@ func DeletePasskeyForSession(identity AuthSessionIdentity) error {
func deletePasskeyWithAuthVersion(userID int, identity *AuthSessionIdentity) error {
if userID == 0 {
return fmt.Errorf("删除失败,请重试")
return errors.New("failed to delete the Passkey, please try again")
}
if err := DB.Transaction(func(tx *gorm.DB) error {
if identity != nil {
+19
View File
@@ -115,6 +115,25 @@ func GetPerfMetricsSummaryBucketsAll(startTs int64, endTs int64, groups []string
return summaries, err
}
// GetRecentPerfMetricSuccessBuckets reads the latest nonempty hours within
// the supplied history window, independently of the summary's reporting window.
func GetRecentPerfMetricSuccessBuckets(modelName string, startTs int64, endTs int64, groups []string, limit int) ([]PerfMetricSummaryBucket, error) {
if modelName == "" || limit <= 0 || (groups != nil && len(groups) == 0) {
return nil, nil
}
const hourBucket = "bucket_ts - (bucket_ts % 3600)"
query := DB.Model(&PerfMetric{}).
Select("model_name, "+hourBucket+" AS bucket_ts, SUM(request_count) AS request_count, SUM(success_count) AS success_count").
Where("model_name = ? AND bucket_ts >= ? AND bucket_ts <= ? AND request_count > 0", modelName, startTs, endTs)
if groups != nil {
query = query.Where(commonGroupCol+" IN ?", groups)
}
var buckets []PerfMetricSummaryBucket
err := query.Group("model_name, " + hourBucket).
Order("bucket_ts DESC").Limit(limit).Find(&buckets).Error
return buckets, err
}
func DeletePerfMetricsBefore(cutoffTs int64) error {
if cutoffTs <= 0 {
return nil
+2 -3
View File
@@ -1,7 +1,6 @@
package model
import (
"fmt"
"strings"
"sync"
"time"
@@ -171,7 +170,7 @@ func loadPricingAdvancedCustomConfigs(enableAbilities []AbilityWithChannel) map[
for _, channelID := range channelIDs {
channel, err := CacheGetChannel(channelID)
if err != nil {
common.SysLog(fmt.Sprintf("load advanced custom channel settings error: channel_id=%d, error=%v", channelID, err))
common.SysLog(common.LogText("load advanced custom channel settings error: channel_id=%d, error=%v", channelID, err))
continue
}
if channel.Type != constant.ChannelTypeAdvancedCustom {
@@ -195,7 +194,7 @@ func updatePricing() {
//modelRatios := common.GetModelRatios()
enableAbilities, err := GetAllEnableAbilityWithChannels()
if err != nil {
common.SysLog(fmt.Sprintf("GetAllEnableAbilityWithChannels error: %v", err))
common.SysLog(common.LogText("GetAllEnableAbilityWithChannels error: %v", err))
return
}
// 预加载模型元数据与供应商一次,避免循环查询
+13
View File
@@ -211,10 +211,23 @@ export function parseTaskResult(){return {};}
require.ErrorContains(t, ValidateModelPricing("profile-image", PricingValues{"billing_setting.billing_expr": `u("seconds")`}), "not declared")
require.ErrorContains(t, ValidateModelPricing("video-alias", PricingValues{"billing_setting.billing_expr": `u("image_count")`}), "not declared")
saved := config.GlobalConfig.ExportAllConfigs()
t.Cleanup(func() { require.NoError(t, config.GlobalConfig.LoadFromDB(saved)) })
require.NoError(t, config.GlobalConfig.LoadFromDB(map[string]string{
"billing_setting.billing_expr": `{"profile-image":"u(\"image_count\")"}`,
billing_setting.PluginBillingExprOption: `{"pricing-profiles::profile-image":"u(\"image_count\") * 2"}`,
}))
updated := strings.Replace(source, `version:"1.0.0"`, `version:"1.1.0"`, 1)
updated = strings.Replace(updated, `image_count:{type:"number",unit:"count"}`, `images:{type:"number",unit:"count"}`, 1)
_, err = jsplugin.DefaultRegistry.Register(updated, jsplugin.Options{})
require.NoError(t, err)
// Stored prices that read a fact the narrowed profile dropped stay saveable
// while unchanged; edited ones must follow the new profile.
stored := PricingValues{"billing_setting.billing_expr": `u("image_count")`, billing_setting.PluginBillingExprOption: map[string]any{key: `u("image_count") * 2`}}
require.NoError(t, ValidateModelPricing("profile-image", stored))
require.ErrorContains(t, ValidateModelPricing("profile-image", PricingValues{"billing_setting.billing_expr": `u("image_count") * 3`}), "not declared")
stored[billing_setting.PluginBillingExprOption] = map[string]any{key: `u("image_count") * 3`}
require.ErrorContains(t, ValidateModelPricing("profile-image", stored), "not declared")
InvalidatePricingCache()
refreshed := pricingByModel(GetPricing())
assert.Equal(t, map[string]jsplugin.UsageFieldSchema{"images": {Type: "number", Unit: "count"}}, refreshed["image-alias"].BillingUsageSchema)
+6 -7
View File
@@ -3,7 +3,6 @@ package model
import (
"context"
"errors"
"fmt"
"github.com/QuantumNous/new-api/common"
"gorm.io/gorm"
@@ -164,7 +163,7 @@ func reserveTokenQuotaDB(id int, quota int) (bool, error) {
// Redis 异常或水合失败时降级为数据库条件更新,保证服务可用。
func TryReserveUserQuota(id int, quota int) (bool, error) {
if quota < 0 {
return false, errors.New("quota 不能为负数!")
return false, errors.New("quota cannot be negative")
}
if quota == 0 {
return true, nil
@@ -181,7 +180,7 @@ func TryReserveUserQuota(id int, quota int) (bool, error) {
}
if err != nil || result == cacheQuotaMiss {
if err != nil {
common.SysLog("user quota cache reserve unavailable, falling back to database: " + err.Error())
common.SysLog(common.LogText("user quota cache reserve unavailable, falling back to database: %s", err.Error()))
}
return reserveUserQuotaDB(id, quota)
}
@@ -191,7 +190,7 @@ func TryReserveUserQuota(id int, quota int) (bool, error) {
if err = persistUserQuotaDelta(id, -quota); err != nil {
compensated, compensateErr := cacheApplyUserQuotaDelta(id, int64(quota))
if compensateErr != nil || compensated != cacheQuotaOK {
common.SysError(fmt.Sprintf("failed to compensate reserved user quota: result=%d error=%v", compensated, compensateErr))
common.SysError(common.LogText("failed to compensate reserved user quota: result=%d error=%v", compensated, compensateErr))
}
return false, err
}
@@ -202,7 +201,7 @@ func TryReserveUserQuota(id int, quota int) (bool, error) {
// tokens skip the balance check but still update remain/used accounting.
func TryReserveTokenQuota(id int, key string, quota int, unlimited bool) (bool, error) {
if quota < 0 {
return false, errors.New("quota 不能为负数!")
return false, errors.New("quota cannot be negative")
}
if quota == 0 {
return true, nil
@@ -222,7 +221,7 @@ func TryReserveTokenQuota(id int, key string, quota int, unlimited bool) (bool,
}
if err != nil || result == cacheQuotaMiss {
if err != nil {
common.SysLog("token quota cache reserve unavailable, falling back to database: " + err.Error())
common.SysLog(common.LogText("token quota cache reserve unavailable, falling back to database: %s", err.Error()))
}
return reserveTokenQuotaDB(id, quota)
}
@@ -232,7 +231,7 @@ func TryReserveTokenQuota(id int, key string, quota int, unlimited bool) (bool,
if err = persistTokenQuotaDelta(id, -quota); err != nil {
compensated, compensateErr := cacheApplyTokenQuotaDelta(id, key, int64(quota))
if compensateErr != nil || compensated != cacheQuotaOK {
common.SysError(fmt.Sprintf("failed to compensate reserved token quota: result=%d error=%v", compensated, compensateErr))
common.SysError(common.LogText("failed to compensate reserved token quota: result=%d error=%v", compensated, compensateErr))
}
return false, err
}
+11 -12
View File
@@ -2,7 +2,6 @@ package model
import (
"errors"
"fmt"
"strconv"
"github.com/QuantumNous/new-api/common"
@@ -126,7 +125,7 @@ func SearchRedemptions(keyword string, status string, startIdx int, num int) (re
func GetRedemptionById(id int) (*Redemption, error) {
if id == 0 {
return nil, errors.New("id 为空!")
return nil, common.NewMessage("ID is empty")
}
redemption := Redemption{Id: id}
var err error = nil
@@ -136,10 +135,10 @@ func GetRedemptionById(id int) (*Redemption, error) {
func Redeem(key string, userId int) (quota int, err error) {
if key == "" {
return 0, errors.New("未提供兑换码")
return 0, errors.New("redemption code not provided")
}
if userId == 0 {
return 0, errors.New("无效的 user id")
return 0, errors.New("invalid user id")
}
redemption := &Redemption{}
@@ -151,13 +150,13 @@ func Redeem(key string, userId int) (quota int, err error) {
err = DB.Transaction(func(tx *gorm.DB) error {
err := lockForUpdate(tx).Where(keyCol+" = ?", key).First(redemption).Error
if err != nil {
return errors.New("无效的兑换码")
return errors.New("invalid redemption code")
}
if redemption.Status != common.RedemptionCodeStatusEnabled {
return errors.New("该兑换码已被使用")
return errors.New("redemption code already used")
}
if redemption.ExpiredTime != 0 && redemption.ExpiredTime < common.GetTimestamp() {
return errors.New("该兑换码已过期")
return errors.New("redemption code expired")
}
// Compare-and-swap on status: only the transaction that flips
// enabled -> used may credit quota, so a concurrent redeem of the
@@ -173,16 +172,16 @@ func Redeem(key string, userId int) (quota int, err error) {
return result.Error
}
if result.RowsAffected == 0 {
return errors.New("该兑换码已被使用")
return errors.New("redemption code already used")
}
return creditTopUpQuota(tx, userId, redemption.Quota, nil)
})
if err != nil {
common.SysError("redemption failed: " + err.Error())
common.SysError(common.LogText("redemption failed: %s", err.Error()))
return 0, ErrRedeemFailed
}
syncCreditUserQuotaCache(userId, redemption.Quota, "redemption")
RecordLog(userId, LogTypeTopup, fmt.Sprintf("通过兑换码充值 %s,兑换码ID %d", logger.LogQuota(redemption.Quota), redemption.Id))
syncCreditUserQuotaCache(userId, redemption.Quota, common.LogText("redemption"))
RecordLog(userId, LogTypeTopup, common.NewMessage("Topped up {{quota}} with a redemption code, code ID {{id}}", map[string]any{"quota": logger.FormatQuota(redemption.Quota), "id": redemption.Id}))
return redemption.Quota, nil
}
@@ -224,7 +223,7 @@ func (redemption *Redemption) Delete() error {
func DeleteRedemptionById(id int) (err error) {
if id == 0 {
return errors.New("id 为空!")
return common.NewMessage("ID is empty")
}
redemption := Redemption{Id: id}
err = DB.Where(redemption).First(&redemption).Error
+27
View File
@@ -24,6 +24,33 @@ func TestTokenRoutingResolution(t *testing.T) {
require.Equal(t, []TokenGroupEntry{{"a", 0}, {"b", 0}, {"c", 1}}, p.Resolve("MODEL-X", nil, allowed))
}
func TestTokenRoutingBlacklistValidation(t *testing.T) {
for _, test := range []struct {
name string
groups []string
valid bool
}{
{"legacy", nil, true},
{"empty", []string{}, true},
{"unavailable group", []string{"removed-group"}, true},
{"empty name", []string{""}, false},
{"virtual auto", []string{"auto"}, false},
{"leading space", []string{" default"}, false},
{"trailing space", []string{"default "}, false},
{"duplicate", []string{"default", "default"}, false},
} {
t.Run(test.name, func(t *testing.T) {
policy := TokenRoutingPolicy{Version: 1, Mode: "smart", ExcludedGroups: test.groups}
err := policy.Validate(5, func(string) bool { return false })
if test.valid {
require.NoError(t, err)
} else {
require.Error(t, err)
}
})
}
}
func TestTokenRoutingLegacy(t *testing.T) {
for _, tc := range []struct {
token Token
+41 -35
View File
@@ -499,7 +499,7 @@ func CreateUserSubscriptionFromPlanTx(tx *gorm.DB, userId int, plan *Subscriptio
return nil, err
}
if count >= int64(plan.MaxPurchasePerUser) {
return nil, errors.New("已达到该套餐购买上限")
return nil, common.NewMessage("Purchase limit for this plan has been reached")
}
}
nowUnix := GetDBTimestamp()
@@ -559,7 +559,7 @@ func CreateUserSubscriptionFromPlanTx(tx *gorm.DB, userId int, plan *Subscriptio
func refreshSubscriptionUserGroupCache(userId int, operation string) {
if err := RefreshUserGroupCache(userId); err != nil {
common.SysError(fmt.Sprintf("failed to refresh user group cache after %s for user %d: %v", operation, userId, err))
common.SysError(common.LogText("failed to refresh user group cache after %s for user %d: %v", operation, userId, err))
}
}
@@ -637,11 +637,14 @@ func CompleteSubscriptionOrder(tradeNo string, providerPayload string, expectedP
return err
}
if upgradeGroup != "" && logUserId > 0 {
refreshSubscriptionUserGroupCache(logUserId, "subscription payment completion")
refreshSubscriptionUserGroupCache(logUserId, common.LogText("subscription payment completion"))
}
if logUserId > 0 {
msg := fmt.Sprintf("订阅购买成功,套餐: %s,支付金额: %.2f,支付方式: %s", logPlanTitle, logMoney, logPaymentMethod)
RecordLog(logUserId, LogTypeTopup, msg)
RecordLog(logUserId, LogTypeTopup, common.NewMessage("Subscription purchased, plan: {{plan}}, payment amount: {{amount}}, payment method: {{method}}", map[string]any{
"plan": logPlanTitle,
"amount": fmt.Sprintf("%.2f", logMoney),
"method": logPaymentMethod,
}))
}
return nil
}
@@ -708,13 +711,13 @@ func ExpireSubscriptionOrder(tradeNo string, expectedPaymentProvider string) err
}
// Admin bind (no payment). Creates a UserSubscription from a plan.
func AdminBindSubscription(userId int, planId int, sourceNote string) (string, error) {
func AdminBindSubscription(userId int, planId int, sourceNote string) (*common.Message, error) {
if userId <= 0 || planId <= 0 {
return "", errors.New("invalid userId or planId")
return nil, errors.New("invalid userId or planId")
}
plan, err := GetSubscriptionPlanById(planId)
if err != nil {
return "", err
return nil, err
}
groupChanged := false
err = DB.Transaction(func(tx *gorm.DB) error {
@@ -730,13 +733,13 @@ func AdminBindSubscription(userId int, planId int, sourceNote string) (string, e
return err
})
if err != nil {
return "", err
return nil, err
}
if groupChanged {
refreshSubscriptionUserGroupCache(userId, "admin subscription creation")
return fmt.Sprintf("用户分组将升级到 %s", plan.UpgradeGroup), nil
refreshSubscriptionUserGroupCache(userId, common.LogText("admin subscription creation"))
return common.NewMessage("User group will be upgraded to {{group}}", map[string]any{"group": plan.UpgradeGroup}), nil
}
return "", nil
return nil, nil
}
func calcSubscriptionBalanceQuota(priceAmount float64) (int, error) {
@@ -744,7 +747,7 @@ func calcSubscriptionBalanceQuota(priceAmount float64) (int, error) {
return 0, nil
}
if common.QuotaPerUnit <= 0 {
return 0, errors.New("额度单位配置错误")
return 0, common.NewMessage("Quota unit is misconfigured")
}
quota := decimal.NewFromFloat(priceAmount).
Mul(decimal.NewFromFloat(common.QuotaPerUnit)).
@@ -768,13 +771,13 @@ func PurchaseSubscriptionWithBalance(userId int, planId int) error {
return err
}
if !plan.Enabled {
return errors.New("套餐未启用")
return common.NewMessage("Subscription plan is not enabled")
}
if plan.PriceAmount < 0 {
return errors.New("套餐价格不能为负数")
return common.NewMessage("Price cannot be negative")
}
if plan.AllowBalancePay != nil && !*plan.AllowBalancePay {
return errors.New("该套餐不允许使用余额兑换")
return common.NewMessage("This plan does not allow balance redemption")
}
requiredQuota, err := calcSubscriptionBalanceQuota(plan.PriceAmount)
@@ -787,7 +790,7 @@ func PurchaseSubscriptionWithBalance(userId int, planId int) error {
return err
}
if requiredQuota > 0 && user.Quota < requiredQuota {
return errors.New("余额不足")
return common.NewMessage("Insufficient balance")
}
if requiredQuota > 0 {
if err := tx.Model(&User{}).Where("id = ?", userId).
@@ -833,14 +836,17 @@ func PurchaseSubscriptionWithBalance(userId int, planId int) error {
if chargedQuota > 0 {
if err := cacheDecrUserQuota(userId, int64(chargedQuota)); err != nil {
common.SysLog("failed to decrease user quota cache after subscription balance purchase: " + err.Error())
common.SysLog(common.LogText("failed to decrease user quota cache after subscription balance purchase: %s", err.Error()))
}
}
if upgradeGroup != "" {
refreshSubscriptionUserGroupCache(userId, "subscription balance purchase")
refreshSubscriptionUserGroupCache(userId, common.LogText("subscription balance purchase"))
}
msg := fmt.Sprintf("使用余额购买订阅成功,套餐: %s,支付金额: %.2f,扣除额度: %d", logPlanTitle, logMoney, chargedQuota)
RecordLog(userId, LogTypeTopup, msg)
RecordLog(userId, LogTypeTopup, common.NewMessage("Subscription purchased with balance, plan: {{plan}}, payment amount: {{amount}}, quota deducted: {{quota}}", map[string]any{
"plan": logPlanTitle,
"amount": fmt.Sprintf("%.2f", logMoney),
"quota": chargedQuota,
}))
return nil
}
@@ -924,9 +930,9 @@ func buildSubscriptionSummaries(subs []UserSubscription) []SubscriptionSummary {
}
// AdminInvalidateUserSubscription marks a user subscription as cancelled and ends it immediately.
func AdminInvalidateUserSubscription(userSubscriptionId int) (string, error) {
func AdminInvalidateUserSubscription(userSubscriptionId int) (*common.Message, error) {
if userSubscriptionId <= 0 {
return "", errors.New("invalid userSubscriptionId")
return nil, errors.New("invalid userSubscriptionId")
}
now := common.GetTimestamp()
cacheGroup := ""
@@ -957,21 +963,21 @@ func AdminInvalidateUserSubscription(userSubscriptionId int) (string, error) {
return nil
})
if err != nil {
return "", err
return nil, err
}
if cacheGroup != "" && userId > 0 {
refreshSubscriptionUserGroupCache(userId, "admin subscription update")
refreshSubscriptionUserGroupCache(userId, common.LogText("admin subscription update"))
}
if downgradeGroup != "" {
return fmt.Sprintf("用户分组将回退到 %s", downgradeGroup), nil
return common.NewMessage("User group will be reverted to {{group}}", map[string]any{"group": downgradeGroup}), nil
}
return "", nil
return nil, nil
}
// AdminDeleteUserSubscription hard-deletes a user subscription.
func AdminDeleteUserSubscription(userSubscriptionId int) (string, error) {
func AdminDeleteUserSubscription(userSubscriptionId int) (*common.Message, error) {
if userSubscriptionId <= 0 {
return "", errors.New("invalid userSubscriptionId")
return nil, errors.New("invalid userSubscriptionId")
}
now := common.GetTimestamp()
cacheGroup := ""
@@ -998,15 +1004,15 @@ func AdminDeleteUserSubscription(userSubscriptionId int) (string, error) {
return nil
})
if err != nil {
return "", err
return nil, err
}
if cacheGroup != "" && userId > 0 {
refreshSubscriptionUserGroupCache(userId, "admin subscription deletion")
refreshSubscriptionUserGroupCache(userId, common.LogText("admin subscription deletion"))
}
if downgradeGroup != "" {
return fmt.Sprintf("用户分组将回退到 %s", downgradeGroup), nil
return common.NewMessage("User group will be reverted to {{group}}", map[string]any{"group": downgradeGroup}), nil
}
return "", nil
return nil, nil
}
func resetUserSubscriptionTx(tx *gorm.DB, sub *UserSubscription, plan *SubscriptionPlan, now int64, advanceResetTime bool) error {
@@ -1059,7 +1065,7 @@ func adminResetUserSubscriptionsByPlanTx(tx *gorm.DB, userId int, plan *Subscrip
return nil, err
}
if len(subs) == 0 {
return nil, errors.New("该用户没有有效的此套餐订阅")
return nil, common.NewMessage("This user has no active subscription to this plan")
}
for i := range subs {
if err := resetUserSubscriptionTx(tx, &subs[i], plan, now, advanceResetTime); err != nil {
@@ -1228,7 +1234,7 @@ func ExpireDueSubscriptions(limit int) (int, error) {
return expiredCount, err
}
if cacheGroup != "" {
refreshSubscriptionUserGroupCache(userId, "subscription expiration")
refreshSubscriptionUserGroupCache(userId, common.LogText("subscription expiration"))
}
}
return expiredCount, nil
+2 -1
View File
@@ -138,7 +138,8 @@ func TestSubscriptionGroupCacheRefreshFailureDoesNotChangeCommittedResult(t *tes
message, err := AdminBindSubscription(user.Id, plan.Id, "test")
require.NoError(t, err)
assert.Contains(t, message, "pro")
require.NotNil(t, message)
assert.Equal(t, "pro", message.Params["group"])
var updated User
require.NoError(t, DB.First(&updated, user.Id).Error)
+1 -1
View File
@@ -134,7 +134,7 @@ func TestAdminResetUserSubscriptionsByPlanNoActiveMatchReturnsError(t *testing.T
require.Error(t, err)
assert.Nil(t, result)
assert.True(t, strings.Contains(err.Error(), "该用户没有有效的此套餐订阅"))
assert.True(t, strings.Contains(err.Error(), "This user has no active subscription to this plan"))
}
func TestAdminResetPlanSubscriptionsResetsAllActiveUsers(t *testing.T) {
+30 -6
View File
@@ -131,6 +131,11 @@ type TaskPrivateData struct {
PluginState json.RawMessage `json:"plugin_state,omitempty"`
// PollFailures counts consecutive unrecognized or transient poll outcomes.
PollFailures int `json:"poll_failures,omitempty"`
// ResultDiscarded marks an immediate terminal result whose submit route
// declared retainResult: false. The upstream snapshot was never written
// and every retrieval surface treats the task as not found. The zero
// value keeps historical rows retained and retrievable.
ResultDiscarded bool `json:"result_discarded,omitempty"`
}
type TaskExecutionSnapshot struct {
@@ -166,6 +171,12 @@ type TaskBillingContext struct {
TieredSnapshot *billingexpr.BillingSnapshot `json:"tiered_snapshot,omitempty"`
}
// ResultRetrievable reports whether retrieval surfaces (native query routes,
// protocol retrieve endpoints, artifact projection) may serve this task.
func (t *Task) ResultRetrievable() bool {
return !t.PrivateData.ResultDiscarded
}
// GetUpstreamTaskID 获取上游真实 task ID(用于与 provider 通信)
// 旧数据没有 UpstreamTaskID 时,TaskID 本身就是上游 ID
func (t *Task) GetUpstreamTaskID() string {
@@ -202,7 +213,8 @@ func (p TaskPrivateData) Value() (driver.Value, error) {
if p.Key == "" && p.UpstreamTaskID == "" && p.ResultURL == "" &&
p.Execution == nil && p.BillingSource == "" && p.SubscriptionId == 0 &&
p.TokenId == 0 && p.NodeName == "" && p.BillingContext == nil &&
!p.ResponsesBackground && len(p.PluginState) == 0 && p.PollFailures == 0 {
!p.ResponsesBackground && len(p.PluginState) == 0 && p.PollFailures == 0 &&
!p.ResultDiscarded {
return nil, nil
}
// 同 Properties.Value:string 避免 PG simple protocol 的 bytea 编码。
@@ -230,8 +242,11 @@ func InitTask(platform constant.TaskPlatform, relayInfo *commonRelay.RelayInfo)
properties := Properties{}
privateData := TaskPrivateData{}
if relayInfo != nil && relayInfo.ChannelMeta != nil {
// A New API channel may rotate between several gateway tokens, so the
// task keeps the key that submitted it and polls with the same identity.
if relayInfo.ChannelMeta.ChannelType == constant.ChannelTypeGemini ||
relayInfo.ChannelMeta.ChannelType == constant.ChannelTypeVertexAi {
relayInfo.ChannelMeta.ChannelType == constant.ChannelTypeVertexAi ||
relayInfo.ChannelMeta.ChannelType == constant.ChannelTypeNewAPI {
privateData.Key = relayInfo.ChannelMeta.ApiKey
}
if relayInfo.UpstreamModelName != "" {
@@ -293,7 +308,9 @@ func TaskGetAllUserTask(userId int, startIdx int, num int, queryParams SyncTaskQ
}
// 获取数据
err = query.Omit("channel_id").Order("id desc").Limit(num).Offset(startIdx).Find(&tasks).Error
// Task lists never render the persisted upstream snapshot; the dashboard
// loads media through the artifacts endpoint instead.
err = query.Omit("channel_id", "data").Order("id desc").Limit(num).Offset(startIdx).Find(&tasks).Error
if err != nil {
return nil
}
@@ -338,7 +355,7 @@ func TaskGetAllTasks(startIdx int, num int, queryParams SyncTaskQueryParams) []*
}
// 获取数据
err = query.Order("id desc").Limit(num).Offset(startIdx).Find(&tasks).Error
err = query.Omit("data").Order("id desc").Limit(num).Offset(startIdx).Find(&tasks).Error
if err != nil {
return nil
}
@@ -469,8 +486,15 @@ func (Task *Task) Insert() error {
return Task.InsertWithContext(context.Background())
}
func (Task *Task) InsertWithContext(ctx context.Context) error {
return DB.WithContext(ctx).Create(Task).Error
// InsertWithContext creates the row. omitColumns are left out of the INSERT
// (for example "data" when the submit route discards the upstream snapshot)
// while the in-memory task keeps its values for presentation.
func (Task *Task) InsertWithContext(ctx context.Context, omitColumns ...string) error {
tx := DB.WithContext(ctx)
if len(omitColumns) > 0 {
tx = tx.Omit(omitColumns...)
}
return tx.Create(Task).Error
}
type taskSnapshot struct {
+2
View File
@@ -59,6 +59,7 @@ func TestMain(m *testing.M) {
&SystemInstance{},
&SystemTask{},
&SystemTaskLock{},
&UserAccessToken{},
); err != nil {
panic("failed to migrate: " + err.Error())
}
@@ -77,6 +78,7 @@ func truncateTables(t *testing.T) {
DB.Exec("DELETE FROM two_fa_backup_codes")
DB.Exec("DELETE FROM two_fas")
DB.Exec("DELETE FROM tokens")
DB.Exec("DELETE FROM user_access_tokens")
DB.Exec("DELETE FROM user_oauth_bindings")
DB.Exec("DELETE FROM users")
DB.Exec("DELETE FROM logs")
+4 -5
View File
@@ -1,7 +1,6 @@
package model
import (
"fmt"
"sync"
"sync/atomic"
"time"
@@ -97,7 +96,7 @@ func buildTaskAliasView(generation *jsplugin.RoutingGeneration) *taskAliasView {
Where("status = ?", common.ChannelStatusEnabled).
Find(&channels).Error
if err != nil {
common.SysError(fmt.Sprintf("rebuild task alias view: %s", err.Error()))
common.SysError(common.LogText("rebuild task alias view: %s", err.Error()))
return view
}
@@ -110,7 +109,7 @@ func buildTaskAliasView(generation *jsplugin.RoutingGeneration) *taskAliasView {
}
modelMap := make(map[string]string)
if err := common.UnmarshalJsonStr(mappingJSON, &modelMap); err != nil {
common.SysError(fmt.Sprintf("task alias view: channel %d model_mapping: %s", channel.Id, err.Error()))
common.SysError(common.LogText("task alias view: channel %d model_mapping: %s", channel.Id, err.Error()))
continue
}
inModels := make(map[string]struct{})
@@ -129,7 +128,7 @@ func buildTaskAliasView(generation *jsplugin.RoutingGeneration) *taskAliasView {
}
tail, cyclic := followChannelModelMapping(modelMap, alias)
if cyclic {
common.SysError(fmt.Sprintf("task alias mapping cycle dropped: channel=%d key=%q", channel.Id, alias))
common.SysError(common.LogText("task alias mapping cycle dropped: channel=%d key=%q", channel.Id, alias))
continue
}
declared, ok := generation.CanonicalModel(tail)
@@ -168,7 +167,7 @@ func buildTaskAliasView(generation *jsplugin.RoutingGeneration) *taskAliasView {
for key := range draft.byPlugin {
pluginKeys = append(pluginKeys, key)
}
common.SysLog(fmt.Sprintf("task model alias %q dropped: maps to multiple plugins %v", alias, pluginKeys))
common.SysLog(common.LogText("task model alias %q dropped: maps to multiple plugins %v", alias, pluginKeys))
continue
}
var pluginKey, declared string
+75 -19
View File
@@ -4,6 +4,7 @@ import (
"crypto/sha256"
"encoding/hex"
"errors"
"slices"
"sort"
"time"
@@ -11,22 +12,25 @@ import (
"github.com/QuantumNous/new-api/constant"
"gorm.io/gorm"
"gorm.io/gorm/clause"
"gorm.io/gorm/schema"
)
type TaskPluginChannelRef struct {
Id int `json:"id"`
Name string `json:"name"`
Type int `json:"type"`
}
func GetTaskPluginUsage(key string) ([]TaskPluginChannelRef, int64, error) {
var channels []Channel
if err := DB.Where("type = ? AND status = ?", constant.ChannelTypeTaskPlugin, common.ChannelStatusEnabled).Find(&channels).Error; err != nil {
boundTypes := []int{constant.ChannelTypeTaskPlugin, constant.ChannelTypeNewAPI}
if err := DB.Where("type IN ? AND status = ?", boundTypes, common.ChannelStatusEnabled).Find(&channels).Error; err != nil {
return nil, 0, err
}
refs := make([]TaskPluginChannelRef, 0)
for _, channel := range channels {
if channel.GetSetting().TaskPluginKey == key {
refs = append(refs, TaskPluginChannelRef{Id: channel.Id, Name: channel.Name})
if channel.GetSetting().BindsTaskPlugin(key) {
refs = append(refs, TaskPluginChannelRef{Id: channel.Id, Name: channel.Name, Type: channel.Type})
}
}
var inFlight int64
@@ -34,25 +38,60 @@ func GetTaskPluginUsage(key string) ([]TaskPluginChannelRef, int64, error) {
return refs, inFlight, err
}
// UnbindTaskPlugin removes one plugin from a channel's bindings and reports
// whether the channel changed. A New API gateway channel keeps serving its other
// plugins and its ordinary traffic.
func UnbindTaskPlugin(channelID int, key string) (bool, error) {
channel, err := GetChannelById(channelID, false)
if err != nil {
return false, err
}
setting := channel.GetSetting()
if !setting.BindsTaskPlugin(key) {
return false, nil
}
if setting.TaskPluginKey == key {
setting.TaskPluginKey = ""
}
setting.TaskExtendPluginKeys = slices.DeleteFunc(setting.TaskExtendPluginKeys, func(bound string) bool { return bound == key })
channel.SetSetting(setting)
return true, DB.Model(&Channel{}).Where("id = ?", channelID).Update("setting", channel.Setting).Error
}
// LongText is a string column sized for plugin payloads on every supported
// database: longtext on MySQL, text on PostgreSQL and SQLite. A bare
// `type:text` tag stops at 64 KiB on MySQL. A `size:` tag above that becomes
// varchar(N) on PostgreSQL, which rejects sizes past 10485760, and a NOT NULL
// column declared that way is re-altered by AutoMigrate on every MySQL start
// because the reported column length never equals N. Neither type chosen here
// carries a length, so an up-to-date column is left alone, and MySQL widens a
// shipped text column once, without data loss.
type LongText string
func (LongText) GormDBDataType(db *gorm.DB, _ *schema.Field) string {
if db.Dialector.Name() == string(common.DatabaseTypeMySQL) {
return "longtext"
}
return "text"
}
type TaskPlugin struct {
Id int64 `json:"id"`
Key string `json:"key" gorm:"size:128;not null;uniqueIndex:uk_task_plugin_key_version,priority:1"`
APIVersion int `json:"api_version" gorm:"not null"`
Version string `json:"version" gorm:"size:64;not null;uniqueIndex:uk_task_plugin_key_version,priority:2"`
Source string `json:"source" gorm:"type:text;not null"`
SourceHash string `json:"source_hash" gorm:"size:64;not null"`
Id int64 `json:"id"`
Key string `json:"key" gorm:"size:128;not null;uniqueIndex:uk_task_plugin_key_version,priority:1"`
APIVersion int `json:"api_version" gorm:"not null"`
Version string `json:"version" gorm:"size:64;not null;uniqueIndex:uk_task_plugin_key_version,priority:2"`
Source LongText `json:"source" gorm:"not null"`
SourceHash string `json:"source_hash" gorm:"size:64;not null"`
// Icon is the plugin logo shipped as a sidecar icon.svg / icon.png next to
// plugin.js, stored as a data URI so one column carries both the media
// type and the bytes. It never travels inside list or detail JSON; the UI
// loads it through GET /api/plugin/task/:key/icon. size matches the
// 512 KiB icon cap and makes GORM emit mediumtext on MySQL (a bare TEXT
// column there holds only 64 KiB), varchar(524288) on PostgreSQL, and text
// on SQLite.
Icon string `json:"-" gorm:"size:524288"`
Enabled bool `json:"enabled" gorm:"not null"`
Active bool `json:"active" gorm:"not null;index"`
CreatedAt int64 `json:"created_at" gorm:"not null"`
Remark string `json:"remark" gorm:"type:text"`
// loads it through GET /api/plugin/task/:key/icon. LongText keeps the
// 512 KiB icon cap storable on MySQL, where a bare TEXT holds only 64 KiB.
Icon LongText `json:"-"`
Enabled bool `json:"enabled" gorm:"not null"`
Active bool `json:"active" gorm:"not null;index"`
CreatedAt int64 `json:"created_at" gorm:"not null"`
Remark string `json:"remark" gorm:"type:text"`
}
// HasIcon reports whether this version ships a logo.
@@ -124,12 +163,17 @@ func GetTaskPluginVersion(key, version string) (*TaskPlugin, error) {
return &plugin, nil
}
// ListActiveTaskPlugins returns the enabled override rows without their
// source and icon payloads; see GetTaskPluginSyncSnapshot.
func ListActiveTaskPlugins() ([]TaskPlugin, error) {
snapshot, err := GetTaskPluginSyncSnapshot()
return snapshot.Plugins, err
}
type TaskPluginSyncSnapshot struct {
// Plugins are the enabled active overrides with Source and Icon left
// empty. Sync compares SourceHash against what it already compiled and
// loads source through GetTaskPluginSource only for the rows that changed.
Plugins []TaskPlugin
Revision string
}
@@ -137,9 +181,11 @@ type TaskPluginSyncSnapshot struct {
// GetTaskPluginSyncSnapshot returns the enabled override set together with a
// deterministic revision of every active database override. Nodes can compare
// the revision even though their local routing-generation counters differ.
// The query skips the source and icon columns: every node polls this every
// 30 seconds, and plugin sources may be several MiB each.
func GetTaskPluginSyncSnapshot() (TaskPluginSyncSnapshot, error) {
var activePlugins []TaskPlugin
if err := DB.Where(&TaskPlugin{Active: true}).
if err := DB.Omit("source", "icon").Where(&TaskPlugin{Active: true}).
Order(clause.OrderByColumn{Column: clause.Column{Name: "key"}}).
Order(clause.OrderByColumn{Column: clause.Column{Name: "version"}}).
Order(clause.OrderByColumn{Column: clause.Column{Name: "id"}}).
@@ -185,6 +231,16 @@ func GetTaskPluginSyncSnapshot() (TaskPluginSyncSnapshot, error) {
}, nil
}
// GetTaskPluginSource loads one override's JavaScript by row id. Rows never
// change source in place (SaveTaskPlugin rejects a different source for the
// same key and version), so the id from a sync snapshot identifies exactly the
// text whose SourceHash that snapshot reported.
func GetTaskPluginSource(id int64) (LongText, error) {
var plugin TaskPlugin
err := DB.Select("source").Where(&TaskPlugin{Id: id}).Take(&plugin).Error
return plugin.Source, err
}
func ActivateTaskPlugin(key, version string) error {
return DB.Transaction(func(tx *gorm.DB) error {
var target TaskPlugin
+57
View File
@@ -80,3 +80,60 @@ func TestSharedPluginKeysFilterBothChannelSources(t *testing.T) {
assert.Equal(t, []int{910001, 910002}, kept)
assert.Empty(t, emptied)
}
func TestNewAPIChannelServesExtendedTaskPlugins(t *testing.T) {
truncateTables(t)
priority := int64(0)
weight := uint(1)
baseURL := "https://gateway.example"
extended := `{"task_extend_plugin_keys":["alpha","gamma"]}`
single := `{"task_plugin_key":"delta"}`
channels := []Channel{
{Id: 920001, Type: constant.ChannelTypeNewAPI, Status: common.ChannelStatusEnabled, Name: "gateway", Models: "shared,chat", Group: "default", Priority: &priority, Weight: &weight, BaseURL: &baseURL, Setting: &extended},
{Id: 920002, Type: constant.ChannelTypeNewAPI, Status: common.ChannelStatusEnabled, Name: "gateway-single", Models: "shared", Group: "default", Priority: &priority, Weight: &weight, BaseURL: &baseURL, Setting: &single},
}
for i := range channels {
require.NoError(t, channels[i].Insert())
}
selected, err := GetChannel("default", "shared", 0, identityFilters("alpha", nil))
require.NoError(t, err)
require.NotNil(t, selected)
assert.Equal(t, "gateway", selected.Name)
selected, err = GetChannel("default", "shared", 0, identityFilters("delta", nil))
require.NoError(t, err)
require.NotNil(t, selected)
assert.Equal(t, "gateway-single", selected.Name, "a single task_plugin_key still binds a New API channel")
selected, err = GetChannel("default", "shared", 0, identityFilters("beta", nil))
require.NoError(t, err)
assert.Nil(t, selected, "unbound plugins never reach the gateway channel")
shared := []dto.ChannelFilter{{Kind: dto.FilterTaskPluginIdentity, TaskPluginKey: "beta", TaskPluginKeys: []string{"beta", "gamma"}}}
selected, err = GetChannel("default", "shared", 0, shared)
require.NoError(t, err)
require.NotNil(t, selected)
assert.Equal(t, "gateway", selected.Name, "a bound shared-model candidate admits the channel")
selected, err = GetChannel("default", "chat", 0, identityFilters("", nil))
require.NoError(t, err)
require.NotNil(t, selected)
assert.Equal(t, "gateway", selected.Name, "requests without a pinned plugin keep using the gateway")
}
func TestModelModifierNamesRouteWithAndWithoutMemoryCache(t *testing.T) {
truncateTables(t)
originalMemoryCache := common.MemoryCacheEnabled
t.Cleanup(func() { common.MemoryCacheEnabled = originalMemoryCache; InitChannelCache() })
channel := Channel{Id: 930001, Type: constant.ChannelTypeDeepSeek, Status: common.ChannelStatusEnabled, Name: "deepseek", Models: "deepseek-flash", Group: "default"}
require.NoError(t, channel.Insert())
for _, memoryCache := range []bool{false, true} {
common.MemoryCacheEnabled = memoryCache
InitChannelCache()
selected, err := GetRandomSatisfiedChannel("default", "deepseek-flash@effort:max", 0, nil)
require.NoError(t, err)
require.NotNil(t, selected, "memory cache %v", memoryCache)
assert.Equal(t, "deepseek", selected.Name)
selected, err = GetRandomSatisfiedChannel("default", "deepseek-pro@effort:max", 0, nil)
require.NoError(t, err)
assert.Nil(t, selected, "memory cache %v", memoryCache)
}
}
+27 -27
View File
@@ -192,20 +192,20 @@ func sanitizeLikePattern(input string) (string, error) {
func validateLikePattern(input string) error {
// 1. 连续的 % 直接拒绝
if strings.Contains(input, "%%") {
return errors.New("搜索模式中不允许包含连续的 % 通配符")
return common.NewMessage("The search pattern cannot contain consecutive % wildcards")
}
// 2. 统计 % 数量,不得超过 2
count := strings.Count(input, "%")
if count > 2 {
return errors.New("搜索模式中最多允许包含 2 个 % 通配符")
return common.NewMessage("The search pattern can contain at most 2 % wildcards")
}
// 3. 含 % 时,去掉 % 后关键词长度必须 >= 2
if count > 0 {
stripped := strings.ReplaceAll(input, "%", "")
if len(stripped) < 2 {
return errors.New("使用模糊搜索时,关键词长度至少为 2 个字符")
return common.NewMessage("Fuzzy search keywords must be at least 2 characters long")
}
}
@@ -233,11 +233,11 @@ func SearchUserTokens(userId int, keyword string, token string, offset int, limi
if hasFuzzy {
count, err := CountUserTokens(userId)
if err != nil {
common.SysLog("failed to count user tokens: " + err.Error())
return nil, 0, errors.New("获取令牌数量失败")
common.SysLog(common.LogText("failed to count user tokens: %s", err.Error()))
return nil, 0, common.NewMessage("Failed to get the token count")
}
if int(count) > maxTokens {
return nil, 0, errors.New("令牌数量超过上限,仅允许精确搜索,请勿使用 % 通配符")
return nil, 0, common.NewMessage("Token count exceeds the limit. Only exact search is allowed, do not use the % wildcard")
}
}
@@ -262,15 +262,15 @@ func SearchUserTokens(userId int, keyword string, token string, offset int, limi
// 先查匹配总数(用于分页,受 maxTokens 上限保护,避免全表 COUNT)
err = baseQuery.Limit(maxTokens).Count(&total).Error
if err != nil {
common.SysError("failed to count search tokens: " + err.Error())
return nil, 0, errors.New("搜索令牌失败")
common.SysError(common.LogText("failed to count search tokens: %s", err.Error()))
return nil, 0, common.NewMessage("Failed to search tokens")
}
// 再分页查数据
err = baseQuery.Order("id desc").Offset(offset).Limit(limit).Find(&tokens).Error
if err != nil {
common.SysError("failed to search tokens: " + err.Error())
return nil, 0, errors.New("搜索令牌失败")
common.SysError(common.LogText("failed to search tokens: %s", err.Error()))
return nil, 0, common.NewMessage("Failed to search tokens")
}
return tokens, total, nil
}
@@ -291,7 +291,7 @@ func ValidateUserToken(key string) (token *Token, err error) {
token.Status = common.TokenStatusExpired
err := token.SelectUpdate()
if err != nil {
common.SysLog("failed to update token status" + err.Error())
common.SysLog(common.LogText("failed to update token status%s", err.Error()))
}
}
return token, ErrTokenInvalid
@@ -301,14 +301,14 @@ func ValidateUserToken(key string) (token *Token, err error) {
token.Status = common.TokenStatusExhausted
err := token.SelectUpdate()
if err != nil {
common.SysLog("failed to update token status" + err.Error())
common.SysLog(common.LogText("failed to update token status%s", err.Error()))
}
}
return token, ErrTokenInvalid
}
return token, nil
}
common.SysLog("ValidateUserToken: failed to get token: " + err.Error())
common.SysLog(common.LogText("ValidateUserToken: failed to get token: %s", err.Error()))
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, ErrTokenInvalid
}
@@ -317,7 +317,7 @@ func ValidateUserToken(key string) (token *Token, err error) {
func GetTokenByIds(id int, userId int) (*Token, error) {
if id == 0 || userId == 0 {
return nil, errors.New("id 或 userId 为空!")
return nil, common.NewMessage("Token ID or user ID is empty")
}
token := Token{Id: id, UserId: userId}
var err error = nil
@@ -327,7 +327,7 @@ func GetTokenByIds(id int, userId int) (*Token, error) {
func GetTokenById(id int) (*Token, error) {
if id == 0 {
return nil, errors.New("id 为空!")
return nil, errors.New("token id is empty")
}
token := Token{Id: id}
var err error = nil
@@ -352,7 +352,7 @@ func GetTokenByKey(key string, fromDB bool) (token *Token, err error) {
// 冷缓存时用数据库快照初始化;已存在的哈希只刷新 TTL,
// 避免快照覆盖 Redis 中已被原子预扣的余额。初始化失败不影响本次读取。
if _, cacheErr := cacheInitToken(*token); cacheErr != nil {
common.SysLog("failed to init token cache: " + cacheErr.Error())
common.SysLog(common.LogText("failed to init token cache: %s", cacheErr.Error()))
}
}
return token, nil
@@ -368,7 +368,7 @@ func (token *Token) Insert() error {
func (token *Token) Update() (err error) {
// 写库前失效缓存并设置 fence,防止并发读者把过期快照重新写回缓存。
if cacheErr := invalidateTokenCacheForMutation(token.Key); cacheErr != nil {
common.SysLog("failed to invalidate token cache before update: " + cacheErr.Error())
common.SysLog(common.LogText("failed to invalidate token cache before update: %s", cacheErr.Error()))
}
return DB.Model(token).Select("name", "status", "expired_time", "remain_quota", "unlimited_quota",
"model_limits_enabled", "model_limits", "allow_ips", "group", "cross_group_retry", "astra_auto_switch", "astra_detection_enabled", "auto_groups", "routing_policy").Updates(token).Error
@@ -376,7 +376,7 @@ func (token *Token) Update() (err error) {
func (token *Token) SelectUpdate() (err error) {
if cacheErr := invalidateTokenCacheForMutation(token.Key); cacheErr != nil {
common.SysLog("failed to invalidate token cache before status update: " + cacheErr.Error())
common.SysLog(common.LogText("failed to invalidate token cache before status update: %s", cacheErr.Error()))
}
// This can update zero values
return DB.Model(token).Select("accessed_time", "status").Updates(token).Error
@@ -384,7 +384,7 @@ func (token *Token) SelectUpdate() (err error) {
func (token *Token) Delete() (err error) {
if cacheErr := invalidateTokenCacheForMutation(token.Key); cacheErr != nil {
common.SysLog("failed to invalidate token cache before delete: " + cacheErr.Error())
common.SysLog(common.LogText("failed to invalidate token cache before delete: %s", cacheErr.Error()))
}
return DB.Delete(token).Error
}
@@ -422,7 +422,7 @@ func DisableModelLimits(tokenId int) error {
func DeleteTokenById(id int, userId int) (err error) {
// Why we need userId here? In case user want to delete other's token.
if id == 0 || userId == 0 {
return errors.New("id 或 userId 为空!")
return common.NewMessage("Token ID or user ID is empty")
}
token := Token{Id: id, UserId: userId}
err = DB.Where(token).First(&token).Error
@@ -434,14 +434,14 @@ func DeleteTokenById(id int, userId int) (err error) {
func IncreaseTokenQuota(tokenId int, key string, quota int) (err error) {
if quota < 0 {
return errors.New("quota 不能为负数!")
return errors.New("quota cannot be negative")
}
if common.RedisEnabled {
gopool.Go(func() {
// 守卫式增量:哈希不存在时跳过,由下次读取从数据库水合,
// 绝不创建只有配额字段的残缺哈希。
if _, err := cacheApplyTokenQuotaDelta(tokenId, key, int64(quota)); err != nil {
common.SysLog("failed to increase token quota: " + err.Error())
common.SysLog(common.LogText("failed to increase token quota: %s", err.Error()))
}
})
}
@@ -465,12 +465,12 @@ func increaseTokenQuota(id int, quota int) (err error) {
func DecreaseTokenQuota(id int, key string, quota int) (err error) {
if quota < 0 {
return errors.New("quota 不能为负数!")
return errors.New("quota cannot be negative")
}
if common.RedisEnabled {
gopool.Go(func() {
if _, err := cacheApplyTokenQuotaDelta(id, key, int64(-quota)); err != nil {
common.SysLog("failed to decrease token quota: " + err.Error())
common.SysLog(common.LogText("failed to decrease token quota: %s", err.Error()))
}
})
}
@@ -502,7 +502,7 @@ func CountUserTokens(userId int) (int64, error) {
// BatchDeleteTokens 删除指定用户的一组令牌,返回成功删除数量
func BatchDeleteTokens(ids []int, userId int) (int, error) {
if len(ids) == 0 {
return 0, errors.New("ids 不能为空!")
return 0, common.NewMessage("Token IDs cannot be empty")
}
tx := DB.Begin()
@@ -513,7 +513,7 @@ func BatchDeleteTokens(ids []int, userId int) (int, error) {
return 0, err
}
if err := invalidateTokensCache(tokens); err != nil {
common.SysLog("failed to invalidate token cache before batch delete: " + err.Error())
common.SysLog(common.LogText("failed to invalidate token cache before batch delete: %s", err.Error()))
}
if err := tx.Where("user_id = ? AND id IN (?)", userId, ids).Delete(&Token{}).Error; err != nil {
@@ -544,7 +544,7 @@ func InvalidateUserTokensCache(userId int) error {
return nil
}
if userId <= 0 {
return errors.New("userId 无效")
return errors.New("invalid user id")
}
var tokens []Token
if err := DB.Unscoped().
+28
View File
@@ -46,6 +46,19 @@ func TestTokenAutoGroupsRoundTripThroughRedisHashCache(t *testing.T) {
assert.Equal(t, []string{"vip", "default"}, groups)
}
func TestTokenAllowUserQuotaQueryRoundTripsThroughRedisHashCache(t *testing.T) {
server := useUserCacheMiniRedis(t)
token := Token{Id: 43, UserId: 7, Key: "token-user-quota-cache-key", Name: "user-quota-cache", AllowUserQuotaQuery: true}
initialized, err := cacheInitToken(token)
require.NoError(t, err)
require.Equal(t, 1, initialized)
cached, err := cacheGetTokenByKey(token.Key)
require.NoError(t, err)
assert.True(t, cached.AllowUserQuotaQuery)
assert.Positive(t, server.TTL(getTokenCacheKey(token.Key)))
}
func TestTokenUpdateSynchronouslyNarrowsPreheatedAutoGroupsCache(t *testing.T) {
truncateTables(t)
useUserCacheMiniRedis(t)
@@ -87,3 +100,18 @@ func cacheSetTokenForTest(token Token) error {
_, err := cacheInitToken(token)
return err
}
func TestTokenRoutingBlacklistRoundTripsThroughRedisHashCache(t *testing.T) {
server := useUserCacheMiniRedis(t)
token := Token{Id: 43, UserId: 7, Key: "token-blacklist-cache-key", Name: "blacklist-cache", RoutingPolicy: `{"version":1,"mode":"smart","excluded_groups":["blocked"]}`}
initialized, err := cacheInitToken(token)
require.NoError(t, err)
require.Equal(t, 1, initialized)
cached, err := cacheGetTokenByKey(token.Key)
require.NoError(t, err)
policy, err := cached.GetRoutingPolicy()
require.NoError(t, err)
assert.Equal(t, []string{"blocked"}, policy.ExcludedGroups)
assert.Positive(t, server.TTL(getTokenCacheKey(token.Key)))
}
+13 -1
View File
@@ -20,6 +20,7 @@ type TokenRoutingPolicy struct {
AutoPreference string `json:"auto_preference,omitempty"`
AllowDegraded bool `json:"allow_degraded,omitempty"`
SkipDegradedGroups bool `json:"skip_degraded_groups,omitempty"`
ExcludedGroups []string `json:"excluded_groups,omitempty"`
Version int `json:"version"`
Mode string `json:"mode"`
Groups []TokenGroupEntry `json:"groups"`
@@ -33,7 +34,7 @@ func (p *TokenRoutingPolicy) Validate(maxGroups int, allowed func(string) bool)
if p.Mode == "custom" && len(p.Groups) == 0 {
return fmt.Errorf("a group pool must contain at least one group")
}
if p.Mode == "smart" && (len(p.ModelRules) != 0 || (p.AutoPreference != "" && p.AutoPreference != "cost" && p.AutoPreference != "speed" && p.AutoPreference != "stable")) {
if p.Mode == "smart" && (len(p.ModelRules) != 0 || (p.AutoPreference != "" && p.AutoPreference != "cost" && p.AutoPreference != "speed" && p.AutoPreference != "throughput" && p.AutoPreference != "stable")) {
return fmt.Errorf("invalid automatic routing settings")
}
if p.Mode != "custom" && len(p.Groups) != 0 {
@@ -43,6 +44,13 @@ func (p *TokenRoutingPolicy) Validate(maxGroups int, allowed func(string) bool)
return fmt.Errorf("routing policy exceeds configured limits")
}
pool := map[string]bool{}
excluded := map[string]bool{}
for _, group := range p.ExcludedGroups {
if group == "" || group == "auto" || group != strings.TrimSpace(group) || excluded[group] {
return common.NewMessage("Invalid group blacklist")
}
excluded[group] = true
}
for _, entry := range p.Groups {
if entry.Group == "" || entry.Group == "auto" || entry.Group != strings.TrimSpace(entry.Group) || pool[entry.Group] || entry.Priority < 0 || entry.Priority > 10000 || !allowed(entry.Group) {
return fmt.Errorf("invalid or inaccessible pool group: %s", entry.Group)
@@ -66,6 +74,10 @@ func (p *TokenRoutingPolicy) Validate(maxGroups int, allowed func(string) bool)
return nil
}
func (p *TokenRoutingPolicy) ExcludesGroup(group string) bool {
return p != nil && p.Mode == "smart" && slices.Contains(p.ExcludedGroups, group)
}
// Resolve is shared by routing and the management preview. Model preferences
// precede remaining defaults; matching priorities stay in the same tier.
func (p *TokenRoutingPolicy) Resolve(modelName string, inherited []string, allowed func(string) bool) []TokenGroupEntry {
+54 -53
View File
@@ -43,12 +43,13 @@ const (
)
var (
ErrPaymentMethodMismatch = errors.New("payment method mismatch")
ErrTopUpNotFound = errors.New("topup not found")
ErrTopUpStatusInvalid = errors.New("topup status invalid")
ErrInvalidTopUpQuota = errors.New("invalid top-up quota")
ErrTopUpQuotaLimitExceeded = errors.New("top-up quota limit exceeded")
ErrWalletQuotaLimitExceeded = errors.New("wallet quota limit exceeded")
ErrPaymentMethodMismatch = errors.New("payment method mismatch")
ErrTopUpNotFound = errors.New("topup not found")
ErrTopUpStatusInvalid = errors.New("topup status invalid")
// The quota errors reach the web console, so they carry message keys.
ErrInvalidTopUpQuota error = common.NewMessage("Invalid top-up quota")
ErrTopUpQuotaLimitExceeded error = common.NewMessage("Top-up quota limit exceeded")
ErrWalletQuotaLimitExceeded error = common.NewMessage("Wallet quota limit exceeded")
)
func (topUp *TopUp) Insert() error {
@@ -144,7 +145,7 @@ func GetTopUpByTradeNo(tradeNo string) *TopUp {
func UpdatePendingTopUpStatus(tradeNo string, expectedPaymentProvider string, targetStatus string) error {
if tradeNo == "" {
return errors.New("未提供支付单号")
return errors.New("payment order number not provided")
}
refCol := "`trade_no`"
@@ -175,7 +176,7 @@ func UpdatePendingTopUpStatus(tradeNo string, expectedPaymentProvider string, ta
// 进程内的 LockOrder 只是优化,正确性由本函数的数据库行锁保证。
func RechargeEpay(tradeNo string, actualPaymentMethod string, callerIp string) (alreadyDone bool, err error) {
if tradeNo == "" {
return false, errors.New("未提供支付单号")
return false, errors.New("payment order number not provided")
}
refCol := "`trade_no`"
@@ -218,23 +219,23 @@ func RechargeEpay(tradeNo string, actualPaymentMethod string, callerIp string) (
})
if err != nil {
if !errors.Is(err, ErrTopUpNotFound) && !errors.Is(err, ErrPaymentMethodMismatch) && !errors.Is(err, ErrTopUpStatusInvalid) {
common.SysError("epay topup failed: " + err.Error())
common.SysError(common.LogText("epay topup failed: %s", err.Error()))
}
return false, err
}
if alreadyDone {
return true, nil
}
syncCreditUserQuotaCache(topUp.UserId, quotaToAdd, "epay topup")
syncCreditUserQuotaCache(topUp.UserId, quotaToAdd, common.LogText("epay topup"))
common.SysLog(fmt.Sprintf("易支付充值成功 trade_no=%s user_id=%d quota_to_add=%d money=%.2f", topUp.TradeNo, topUp.UserId, quotaToAdd, topUp.Money))
RecordTopupLog(topUp.UserId, fmt.Sprintf("使用在线充值成功,充值金额: %v,支付金额:%f", logger.LogQuota(quotaToAdd), topUp.Money), callerIp, topUp.PaymentMethod, PaymentProviderEpay)
common.SysLog(common.LogText("Epay top-up succeeded trade_no=%s user_id=%d quota_to_add=%d money=%.2f", topUp.TradeNo, topUp.UserId, quotaToAdd, topUp.Money))
RecordTopupLog(topUp.UserId, common.NewMessage("Online top-up succeeded, amount added: {{quota}}, payment amount: {{amount}}", map[string]any{"quota": logger.FormatQuota(quotaToAdd), "amount": fmt.Sprintf("%f", topUp.Money)}), callerIp, topUp.PaymentMethod, PaymentProviderEpay)
return false, nil
}
func Recharge(referenceId string, customerId string, callerIp string) (err error) {
if referenceId == "" {
return errors.New("未提供支付单号")
return errors.New("payment order number not provided")
}
var quota int
@@ -248,7 +249,7 @@ func Recharge(referenceId string, customerId string, callerIp string) (err error
err = DB.Transaction(func(tx *gorm.DB) error {
err := lockForUpdate(tx).Where(refCol+" = ?", referenceId).First(topUp).Error
if err != nil {
return errors.New("充值订单不存在")
return errors.New("top-up order not found")
}
if topUp.PaymentProvider != PaymentProviderStripe {
@@ -256,7 +257,7 @@ func Recharge(referenceId string, customerId string, callerIp string) (err error
}
if topUp.Status != common.TopUpStatusPending {
return errors.New("充值订单状态错误")
return errors.New("invalid top-up order status")
}
topUp.CompleteTime = common.GetTimestamp()
@@ -278,12 +279,12 @@ func Recharge(referenceId string, customerId string, callerIp string) (err error
})
if err != nil {
common.SysError("topup failed: " + err.Error())
return errors.New("充值失败,请稍后重试")
common.SysError(common.LogText("topup failed: %s", err.Error()))
return errors.New("top-up failed")
}
syncCreditUserQuotaCache(topUp.UserId, quota, "stripe topup")
syncCreditUserQuotaCache(topUp.UserId, quota, common.LogText("stripe topup"))
RecordTopupLog(topUp.UserId, fmt.Sprintf("使用在线充值成功,充值金额: %v,支付金额:%d", logger.FormatQuota(quota), topUp.Amount), callerIp, topUp.PaymentMethod, PaymentMethodStripe)
RecordTopupLog(topUp.UserId, common.NewMessage("Online top-up succeeded, amount added: {{quota}}, payment amount: {{amount}}", map[string]any{"quota": logger.FormatQuota(quota), "amount": topUp.Amount}), callerIp, topUp.PaymentMethod, PaymentMethodStripe)
return nil
}
@@ -389,14 +390,14 @@ func SearchUserTopUps(userId int, keyword string, pageInfo *common.PageInfo) (to
if err = query.Limit(searchTopUpCountHardLimit).Count(&total).Error; err != nil {
tx.Rollback()
common.SysError("failed to count search topups: " + err.Error())
return nil, 0, errors.New("搜索充值记录失败")
common.SysError(common.LogText("failed to count search topups: %s", err.Error()))
return nil, 0, common.NewMessage("Failed to search top-up records")
}
if err = query.Order("id desc").Limit(pageInfo.GetPageSize()).Offset(pageInfo.GetStartIdx()).Find(&topups).Error; err != nil {
tx.Rollback()
common.SysError("failed to search topups: " + err.Error())
return nil, 0, errors.New("搜索充值记录失败")
common.SysError(common.LogText("failed to search topups: %s", err.Error()))
return nil, 0, common.NewMessage("Failed to search top-up records")
}
if err = tx.Commit().Error; err != nil {
@@ -429,14 +430,14 @@ func SearchAllTopUps(keyword string, pageInfo *common.PageInfo) (topups []*TopUp
if err = query.Limit(searchTopUpCountHardLimit).Count(&total).Error; err != nil {
tx.Rollback()
common.SysError("failed to count search topups: " + err.Error())
return nil, 0, errors.New("搜索充值记录失败")
common.SysError(common.LogText("failed to count search topups: %s", err.Error()))
return nil, 0, common.NewMessage("Failed to search top-up records")
}
if err = query.Order("id desc").Limit(pageInfo.GetPageSize()).Offset(pageInfo.GetStartIdx()).Find(&topups).Error; err != nil {
tx.Rollback()
common.SysError("failed to search topups: " + err.Error())
return nil, 0, errors.New("搜索充值记录失败")
common.SysError(common.LogText("failed to search topups: %s", err.Error()))
return nil, 0, common.NewMessage("Failed to search top-up records")
}
if err = tx.Commit().Error; err != nil {
@@ -448,7 +449,7 @@ func SearchAllTopUps(keyword string, pageInfo *common.PageInfo) (topups []*TopUp
// ManualCompleteTopUp 管理员手动完成订单并给用户充值
func ManualCompleteTopUp(tradeNo string, callerIp string) error {
if tradeNo == "" {
return errors.New("未提供订单号")
return common.NewMessage("Order number not provided")
}
refCol := "`trade_no`"
@@ -465,7 +466,7 @@ func ManualCompleteTopUp(tradeNo string, callerIp string) error {
topUp := &TopUp{}
// 行级锁,避免并发补单
if err := lockForUpdate(tx).Where(refCol+" = ?", tradeNo).First(topUp).Error; err != nil {
return errors.New("充值订单不存在")
return common.NewMessage("Top-up order does not exist")
}
// 幂等处理:已成功直接返回
@@ -474,7 +475,7 @@ func ManualCompleteTopUp(tradeNo string, callerIp string) error {
}
if topUp.Status != common.TopUpStatusPending {
return errors.New("订单状态不是待支付,无法补单")
return common.NewMessage("Order is not pending payment and cannot be completed")
}
// 计算应充值额度:
@@ -517,13 +518,13 @@ func ManualCompleteTopUp(tradeNo string, callerIp string) error {
}
// 事务外记录日志,避免阻塞
syncCreditUserQuotaCache(userId, quotaToAdd, "manual topup")
RecordTopupLog(userId, fmt.Sprintf("管理员补单成功,充值金额: %v,支付金额:%f", logger.FormatQuota(quotaToAdd), payMoney), callerIp, paymentMethod, "admin")
syncCreditUserQuotaCache(userId, quotaToAdd, common.LogText("manual topup"))
RecordTopupLog(userId, common.NewMessage("Administrator completed the order, amount added: {{quota}}, payment amount: {{amount}}", map[string]any{"quota": logger.FormatQuota(quotaToAdd), "amount": fmt.Sprintf("%f", payMoney)}), callerIp, paymentMethod, "admin")
return nil
}
func RechargeCreem(referenceId string, customerEmail string, customerName string, callerIp string) (err error) {
if referenceId == "" {
return errors.New("未提供支付单号")
return errors.New("payment order number not provided")
}
var quota int
@@ -537,7 +538,7 @@ func RechargeCreem(referenceId string, customerEmail string, customerName string
err = DB.Transaction(func(tx *gorm.DB) error {
err := lockForUpdate(tx).Where(refCol+" = ?", referenceId).First(topUp).Error
if err != nil {
return errors.New("充值订单不存在")
return errors.New("top-up order not found")
}
if topUp.PaymentProvider != PaymentProviderCreem {
@@ -545,7 +546,7 @@ func RechargeCreem(referenceId string, customerEmail string, customerName string
}
if topUp.Status != common.TopUpStatusPending {
return errors.New("充值订单状态错误")
return errors.New("invalid top-up order status")
}
topUp.CompleteTime = common.GetTimestamp()
@@ -583,19 +584,19 @@ func RechargeCreem(referenceId string, customerEmail string, customerName string
})
if err != nil {
common.SysError("creem topup failed: " + err.Error())
return errors.New("充值失败,请稍后重试")
common.SysError(common.LogText("creem topup failed: %s", err.Error()))
return errors.New("top-up failed")
}
syncCreditUserQuotaCache(topUp.UserId, quota, "creem topup")
syncCreditUserQuotaCache(topUp.UserId, quota, common.LogText("creem topup"))
RecordTopupLog(topUp.UserId, fmt.Sprintf("使用Creem充值成功,充值额度: %v,支付金额:%.2f", quota, topUp.Money), callerIp, topUp.PaymentMethod, PaymentMethodCreem)
RecordTopupLog(topUp.UserId, common.NewMessage("{{provider}} top-up succeeded, amount added: {{quota}}, payment amount: {{amount}}", map[string]any{"provider": "Creem", "quota": quota, "amount": fmt.Sprintf("%.2f", topUp.Money)}), callerIp, topUp.PaymentMethod, PaymentMethodCreem)
return nil
}
func RechargeWaffo(tradeNo string, callerIp string) (err error) {
if tradeNo == "" {
return errors.New("未提供支付单号")
return errors.New("payment order number not provided")
}
var quotaToAdd int
@@ -609,7 +610,7 @@ func RechargeWaffo(tradeNo string, callerIp string) (err error) {
err = DB.Transaction(func(tx *gorm.DB) error {
err := lockForUpdate(tx).Where(refCol+" = ?", tradeNo).First(topUp).Error
if err != nil {
return errors.New("充值订单不存在")
return errors.New("top-up order not found")
}
if topUp.PaymentProvider != PaymentProviderWaffo {
@@ -621,7 +622,7 @@ func RechargeWaffo(tradeNo string, callerIp string) (err error) {
}
if topUp.Status != common.TopUpStatusPending {
return errors.New("充值订单状态错误")
return errors.New("invalid top-up order status")
}
quotaToAdd, err = common.WalletQuotaFromDecimalStrict(
@@ -641,13 +642,13 @@ func RechargeWaffo(tradeNo string, callerIp string) (err error) {
})
if err != nil {
common.SysError("waffo topup failed: " + err.Error())
return errors.New("充值失败,请稍后重试")
common.SysError(common.LogText("waffo topup failed: %s", err.Error()))
return errors.New("top-up failed")
}
syncCreditUserQuotaCache(topUp.UserId, quotaToAdd, "waffo topup")
syncCreditUserQuotaCache(topUp.UserId, quotaToAdd, common.LogText("waffo topup"))
if quotaToAdd > 0 {
RecordTopupLog(topUp.UserId, fmt.Sprintf("Waffo充值成功,充值额度: %v,支付金额: %.2f", logger.FormatQuota(quotaToAdd), topUp.Money), callerIp, topUp.PaymentMethod, PaymentMethodWaffo)
RecordTopupLog(topUp.UserId, common.NewMessage("{{provider}} top-up succeeded, amount added: {{quota}}, payment amount: {{amount}}", map[string]any{"provider": "Waffo", "quota": logger.FormatQuota(quotaToAdd), "amount": fmt.Sprintf("%.2f", topUp.Money)}), callerIp, topUp.PaymentMethod, PaymentMethodWaffo)
}
return nil
@@ -655,7 +656,7 @@ func RechargeWaffo(tradeNo string, callerIp string) (err error) {
func RechargeWaffoPancake(tradeNo string) (err error) {
if tradeNo == "" {
return errors.New("未提供支付单号")
return errors.New("payment order number not provided")
}
var quotaToAdd int
@@ -669,7 +670,7 @@ func RechargeWaffoPancake(tradeNo string) (err error) {
err = DB.Transaction(func(tx *gorm.DB) error {
err := lockForUpdate(tx).Where(refCol+" = ?", tradeNo).First(topUp).Error
if err != nil {
return errors.New("充值订单不存在")
return errors.New("top-up order not found")
}
if topUp.PaymentProvider != PaymentProviderWaffoPancake {
@@ -681,7 +682,7 @@ func RechargeWaffoPancake(tradeNo string) (err error) {
}
if topUp.Status != common.TopUpStatusPending {
return errors.New("充值订单状态错误")
return errors.New("invalid top-up order status")
}
quotaToAdd, err = common.WalletQuotaFromDecimalStrict(
@@ -701,13 +702,13 @@ func RechargeWaffoPancake(tradeNo string) (err error) {
})
if err != nil {
common.SysError("waffo pancake topup failed: " + err.Error())
return errors.New("充值失败,请稍后重试")
common.SysError(common.LogText("waffo pancake topup failed: %s", err.Error()))
return errors.New("top-up failed")
}
syncCreditUserQuotaCache(topUp.UserId, quotaToAdd, "waffo pancake topup")
syncCreditUserQuotaCache(topUp.UserId, quotaToAdd, common.LogText("waffo pancake topup"))
if quotaToAdd > 0 {
RecordLog(topUp.UserId, LogTypeTopup, fmt.Sprintf("Waffo Pancake充值成功,充值额度: %v,支付金额: %.2f", logger.FormatQuota(quotaToAdd), topUp.Money))
RecordLog(topUp.UserId, LogTypeTopup, common.NewMessage("{{provider}} top-up succeeded, amount added: {{quota}}, payment amount: {{amount}}", map[string]any{"provider": "Waffo Pancake", "quota": logger.FormatQuota(quotaToAdd), "amount": fmt.Sprintf("%.2f", topUp.Money)}))
}
return nil
+7 -7
View File
@@ -38,7 +38,7 @@ type TwoFABackupCode struct {
// GetTwoFAByUserId 根据用户ID获取2FA设置
func GetTwoFAByUserId(userId int) (*TwoFA, error) {
if userId == 0 {
return nil, errors.New("用户ID不能为空")
return nil, errors.New("user ID is empty")
}
var twoFA TwoFA
@@ -64,7 +64,7 @@ func IsTwoFAEnabled(userId int) (bool, error) {
func (t *TwoFA) updateUsageState() error {
if t.Id == 0 {
return errors.New("2FA记录ID不能为空")
return errors.New("2FA record ID is empty")
}
return DB.Model(&TwoFA{}).Where("id = ?", t.Id).Updates(map[string]any{
"failed_attempts": t.FailedAttempts,
@@ -83,7 +83,7 @@ func (t *TwoFA) ResetFailedAttempts() error {
// IncrementFailedAttempts 增加失败尝试次数
func (t *TwoFA) IncrementFailedAttempts() error {
if t.Id == 0 {
return errors.New("2FA记录ID不能为空")
return errors.New("2FA record ID is empty")
}
const maxUpdateRetries = 5
@@ -125,7 +125,7 @@ func (t *TwoFA) IncrementFailedAttempts() error {
return nil
}
return errors.New("更新2FA失败次数冲突,请重试")
return errors.New("conflicting updates to the 2FA failure count, please retry")
}
// IsLocked 检查账户是否被锁定
@@ -191,7 +191,7 @@ func replaceBackupCodesWithAuthVersion(userId int, codes []string, identity *Aut
// ValidateBackupCode 验证并使用备用码
func ValidateBackupCode(userId int, code string) (bool, error) {
if !common.ValidateBackupCode(code) {
return false, errors.New("验证码或备用码不正确")
return false, errors.New("the verification code or backup code is incorrect")
}
normalizedCode := common.NormalizeBackupCode(code)
@@ -272,7 +272,7 @@ func disableTwoFAWithAuthVersion(userId int, identity *AuthSessionIdentity) erro
func (t *TwoFA) ValidateTOTPAndUpdateUsage(code string) (bool, error) {
// 检查是否被锁定
if t.IsLocked() {
return false, fmt.Errorf("账户已被锁定,请在%v后重试", t.LockedUntil.Format("2006-01-02 15:04:05"))
return false, fmt.Errorf("the account is locked, retry after %v", t.LockedUntil.Format("2006-01-02 15:04:05"))
}
// 验证TOTP码
@@ -301,7 +301,7 @@ func (t *TwoFA) ValidateTOTPAndUpdateUsage(code string) (bool, error) {
func (t *TwoFA) ValidateBackupCodeAndUpdateUsage(code string) (bool, error) {
// 检查是否被锁定
if t.IsLocked() {
return false, fmt.Errorf("账户已被锁定,请在%v后重试", t.LockedUntil.Format("2006-01-02 15:04:05"))
return false, fmt.Errorf("the account is locked, retry after %v", t.LockedUntil.Format("2006-01-02 15:04:05"))
}
// 验证备用码
+3 -3
View File
@@ -42,7 +42,7 @@ type QuotaDataLogParams struct {
func UpdateQuotaData() {
for {
if common.DataExportEnabled {
common.SysLog("正在更新数据看板数据...")
common.SysLog(common.LogText("updating dashboard data..."))
SaveQuotaDataCache()
}
time.Sleep(time.Duration(common.DataExportInterval) * time.Minute)
@@ -122,7 +122,7 @@ func SaveQuotaDataCache() {
}
}
CacheQuotaData = make(map[string]*QuotaData)
common.SysLog(fmt.Sprintf("保存数据看板数据成功,共保存%d条数据", size))
common.SysLog(common.LogText("saved dashboard data, %d records", size))
}
func increaseQuotaData(quotaData *QuotaData) {
@@ -135,7 +135,7 @@ func increaseQuotaData(quotaData *QuotaData) {
"token_used": gorm.Expr("token_used + ?", quotaData.TokenUsed),
}).Error
if err != nil {
common.SysLog(fmt.Sprintf("increaseQuotaData error: %s", err))
common.SysLog(common.LogText("increaseQuotaData error: %s", err))
}
}
+86 -84
View File
@@ -92,8 +92,8 @@ type User struct {
WeChatId string `json:"wechat_id" gorm:"column:wechat_id;index"`
TelegramId string `json:"telegram_id" gorm:"column:telegram_id;index"`
VerificationCode string `json:"verification_code" gorm:"-:all"` // this field is only for Email verification, don't save it to database!
AccessToken *string `json:"-" gorm:"type:char(32);column:access_token;uniqueIndex"` // this token is for system management
AccessTokenCreatedAt *int64 `json:"-" gorm:"type:bigint;column:access_token_created_at"`
AccessToken *string `json:"-" gorm:"type:char(32);column:access_token;uniqueIndex"` // Deprecated: 旧版面板访问令牌,仅在升级后的过渡期内使用;删除 users.access_token 列时一并移除。
AccessTokenCreatedAt *int64 `json:"-" gorm:"type:bigint;column:access_token_created_at"` // Deprecated: 旧版面板访问令牌,仅在升级后的过渡期内使用;删除 users.access_token 列时一并移除。
Quota int `json:"quota" gorm:"type:int;default:0"`
UsedQuota int `json:"used_quota" gorm:"type:int;default:0;column:used_quota"` // used quota
RequestCount int `json:"request_count" gorm:"type:int;default:0;"` // request number
@@ -130,6 +130,7 @@ func (user *User) ToBaseUser() *UserBase {
return cache
}
// Deprecated: 旧版面板访问令牌,仅在升级后的过渡期内使用;删除 users.access_token 列时一并移除。
func (user *User) GetAccessToken() string {
if user.AccessToken == nil {
return ""
@@ -137,29 +138,14 @@ func (user *User) GetAccessToken() string {
return *user.AccessToken
}
// Deprecated: 旧版面板访问令牌,仅在升级后的过渡期内使用;删除 users.access_token 列时一并移除。
func (user *User) SetAccessToken(token string) {
user.AccessToken = &token
}
// UpdateUserAccessToken rotates a dashboard personal access token without
// writing a stale user snapshot back over concurrently updated fields.
func UpdateUserAccessToken(id int, token string) error {
if id == 0 {
return errors.New("id 为空!")
}
result := DB.Model(&User{}).Where("id = ?", id).Updates(map[string]any{
"access_token": token, "access_token_created_at": common.GetTimestamp(),
})
if result.Error != nil {
return result.Error
}
if result.RowsAffected == 0 {
return gorm.ErrRecordNotFound
}
return nil
}
// RevokeUserAccessToken returns the generation actually revoked under the row lock.
//
// Deprecated: 旧版面板访问令牌,仅在升级后的过渡期内使用;删除 users.access_token 列时一并移除。
func RevokeUserAccessToken(id int) (string, error) {
var tokenRef string
err := DB.Transaction(func(tx *gorm.DB) error {
@@ -181,7 +167,7 @@ func (user *User) GetSetting() dto.UserSetting {
if user.Setting != "" {
err := common.Unmarshal([]byte(user.Setting), &setting)
if err != nil {
common.SysLog("failed to unmarshal setting: " + err.Error())
common.SysLog(common.LogText("failed to unmarshal setting: %s", err.Error()))
}
}
return setting
@@ -190,7 +176,7 @@ func (user *User) GetSetting() dto.UserSetting {
func (user *User) SetSetting(setting dto.UserSetting) {
settingBytes, err := common.Marshal(setting)
if err != nil {
common.SysLog("failed to marshal setting: " + err.Error())
common.SysLog(common.LogText("failed to marshal setting: %s", err.Error()))
return
}
user.Setting = string(settingBytes)
@@ -198,7 +184,7 @@ func (user *User) SetSetting(setting dto.UserSetting) {
func UpdateUserSetting(userId int, setting dto.UserSetting) error {
if userId == 0 {
return errors.New("id 为空!")
return common.NewMessage("ID is empty!")
}
settingBytes, err := common.Marshal(setting)
if err != nil {
@@ -227,7 +213,7 @@ var userBindColumns = map[string]bool{
// 角色、状态、分组只允许通过各自带锁/CAS 的专用方法修改。
func UpdateUserBindColumn(userId int, column string, value string) error {
if userId <= 0 {
return errors.New("id 为空!")
return common.NewMessage("ID is empty!")
}
if !userBindColumns[column] {
return fmt.Errorf("invalid user bind column: %s", column)
@@ -290,7 +276,7 @@ func generateDefaultSidebarConfigForRole(userRole int) string {
// 转换为JSON字符串
configBytes, err := common.Marshal(defaultConfig)
if err != nil {
common.SysLog("生成默认边栏配置失败: " + err.Error())
common.SysLog(common.LogText("failed to generate the default sidebar config: %s", err.Error()))
return ""
}
@@ -520,7 +506,7 @@ func SearchUsers(keyword string, group string, role *int, status *int, startIdx
func GetUserById(id int, selectAll bool) (*User, error) {
if id == 0 {
return nil, errors.New("id 为空!")
return nil, common.NewMessage("ID is empty!")
}
user := User{Id: id}
var err error = nil
@@ -536,7 +522,7 @@ func GetUserById(id int, selectAll bool) (*User, error) {
// query. The password hash and management access token are never selected.
func GetSelfUserById(id int) (*User, error) {
if id == 0 {
return nil, errors.New("id 为空!")
return nil, common.NewMessage("ID is empty!")
}
var profile struct {
User
@@ -556,24 +542,28 @@ func GetSelfUserById(id int) (*User, error) {
func GetUserIdByAffCode(affCode string) (int, error) {
if affCode == "" {
return 0, errors.New("affCode 为空!")
return 0, errors.New("affiliate code is empty")
}
var user User
err := DB.Select("id").First(&user, "aff_code = ?", affCode).Error
return user.Id, err
}
func DeleteUserById(id int) (err error) {
// DeleteUserById soft-deletes a user and returns how many scoped access tokens
// were deleted with it.
func DeleteUserById(id int) (int64, error) {
if id == 0 {
return errors.New("id 为空!")
return 0, common.NewMessage("ID is empty!")
}
user := User{Id: id}
return user.Delete()
}
func HardDeleteUserById(id int) error {
// HardDeleteUserById permanently deletes a user and returns how many scoped
// access tokens were deleted with it.
func HardDeleteUserById(id int) (int64, error) {
if id == 0 {
return errors.New("id 为空!")
return 0, common.NewMessage("ID is empty!")
}
user := User{Id: id}
return user.HardDelete()
@@ -597,7 +587,7 @@ func inviteUser(inviterId int) error {
func (user *User) TransferAffQuotaToQuota(quota int) error {
// 检查quota是否小于最小额度
if float64(quota) < common.QuotaPerUnit {
return fmt.Errorf("转移额度最小为%s!", logger.LogQuota(common.QuotaFromFloat(common.QuotaPerUnit)))
return common.NewMessage("Minimum transfer quota is {{min}}!", map[string]any{"min": logger.FormatQuota(common.QuotaFromFloat(common.QuotaPerUnit))})
}
// 开始数据库事务
@@ -615,7 +605,7 @@ func (user *User) TransferAffQuotaToQuota(quota int) error {
// 再次检查用户的AffQuota是否足够
if user.AffQuota < quota {
return errors.New("邀请额度不足!")
return common.NewMessage("Invitation quota is insufficient!")
}
// 更新用户额度
@@ -720,21 +710,21 @@ func (user *User) finishInsert(inviterId int) {
currentSetting.SidebarModules = defaultSidebarConfig
createdUser.SetSetting(currentSetting)
createdUser.Update(false)
common.SysLog(fmt.Sprintf("为新用户 %s (角色: %d) 初始化边栏配置", createdUser.Username, createdUser.Role))
common.SysLog(common.LogText("initialized the sidebar config for new user %s (role: %d)", createdUser.Username, createdUser.Role))
}
}
if common.QuotaForNewUser > 0 {
RecordLog(user.Id, LogTypeSystem, fmt.Sprintf("新用户注册赠送 %s", logger.LogQuota(common.QuotaForNewUser)))
RecordLog(user.Id, LogTypeSystem, common.NewMessage("New user sign-up bonus {{quota}}", map[string]any{"quota": logger.FormatQuota(common.QuotaForNewUser)}))
}
if inviterId != 0 && operation_setting.IsPaymentComplianceConfirmed() {
if common.QuotaForInvitee > 0 {
_ = IncreaseUserQuota(user.Id, common.QuotaForInvitee, true)
RecordLog(user.Id, LogTypeSystem, fmt.Sprintf("使用邀请码赠送 %s", logger.LogQuota(common.QuotaForInvitee)))
RecordLog(user.Id, LogTypeSystem, common.NewMessage("Invitation code bonus {{quota}}", map[string]any{"quota": logger.FormatQuota(common.QuotaForInvitee)}))
}
if common.QuotaForInviter > 0 {
//_ = IncreaseUserQuota(inviterId, common.QuotaForInviter)
RecordLog(inviterId, LogTypeSystem, fmt.Sprintf("邀请用户赠送 %s", logger.LogQuota(common.QuotaForInviter)))
RecordLog(inviterId, LogTypeSystem, common.NewMessage("Bonus for inviting a user {{quota}}", map[string]any{"quota": logger.FormatQuota(common.QuotaForInviter)}))
_ = inviteUser(inviterId)
}
}
@@ -777,20 +767,20 @@ func (user *User) FinalizeOAuthUserCreation(inviterId int) {
currentSetting.SidebarModules = defaultSidebarConfig
createdUser.SetSetting(currentSetting)
createdUser.Update(false)
common.SysLog(fmt.Sprintf("为新用户 %s (角色: %d) 初始化边栏配置", createdUser.Username, createdUser.Role))
common.SysLog(common.LogText("initialized the sidebar config for new user %s (role: %d)", createdUser.Username, createdUser.Role))
}
}
if common.QuotaForNewUser > 0 {
RecordLog(user.Id, LogTypeSystem, fmt.Sprintf("新用户注册赠送 %s", logger.LogQuota(common.QuotaForNewUser)))
RecordLog(user.Id, LogTypeSystem, common.NewMessage("New user sign-up bonus {{quota}}", map[string]any{"quota": logger.FormatQuota(common.QuotaForNewUser)}))
}
if inviterId != 0 && operation_setting.IsPaymentComplianceConfirmed() {
if common.QuotaForInvitee > 0 {
_ = IncreaseUserQuota(user.Id, common.QuotaForInvitee, true)
RecordLog(user.Id, LogTypeSystem, fmt.Sprintf("使用邀请码赠送 %s", logger.LogQuota(common.QuotaForInvitee)))
RecordLog(user.Id, LogTypeSystem, common.NewMessage("Invitation code bonus {{quota}}", map[string]any{"quota": logger.FormatQuota(common.QuotaForInvitee)}))
}
if common.QuotaForInviter > 0 {
RecordLog(inviterId, LogTypeSystem, fmt.Sprintf("邀请用户赠送 %s", logger.LogQuota(common.QuotaForInviter)))
RecordLog(inviterId, LogTypeSystem, common.NewMessage("Bonus for inviting a user {{quota}}", map[string]any{"quota": logger.FormatQuota(common.QuotaForInviter)}))
_ = inviteUser(inviterId)
}
}
@@ -953,20 +943,21 @@ func (user *User) ClearBinding(bindingType string) error {
return updateUserCache(*user)
}
func (user *User) Delete() error {
func (user *User) Delete() (int64, error) {
return user.delete(nil)
}
func DeleteUserForSession(identity AuthSessionIdentity) error {
func DeleteUserForSession(identity AuthSessionIdentity) (int64, error) {
user := User{Id: identity.UserID}
return user.delete(&identity)
}
func (user *User) delete(identity *AuthSessionIdentity) error {
func (user *User) delete(identity *AuthSessionIdentity) (int64, error) {
if user.Id == 0 {
return errors.New("id 为空!")
return 0, common.NewMessage("ID is empty!")
}
var nextAuthVersion int64
var revokedAccessTokens int64
if err := DB.Transaction(func(tx *gorm.DB) error {
if identity != nil {
if err := ValidateAuthSessionWithTx(tx, *identity); err != nil {
@@ -985,31 +976,40 @@ func (user *User) delete(identity *AuthSessionIdentity) error {
if err != nil {
return err
}
revokedAccessTokens, err = DeleteUserAccessTokensWithTx(tx, user.Id)
if err != nil {
return err
}
return tx.Delete(user).Error
}); err != nil {
return err
return 0, err
}
if err := publishCommittedUserAuthVersion(user.Id, nextAuthVersion); err != nil {
return err
return revokedAccessTokens, err
}
if _, err := RevokeAllUserSessions(user.Id, "user_deleted"); err != nil {
return err
return revokedAccessTokens, err
}
return invalidateUserCache(user.Id)
return revokedAccessTokens, invalidateUserCache(user.Id)
}
func (user *User) HardDelete() error {
func (user *User) HardDelete() (int64, error) {
if user.Id == 0 {
return errors.New("id 为空!")
return 0, common.NewMessage("ID is empty!")
}
var tokens []Token
var deletedAuthVersion int64
var revokedAccessTokens int64
err := DB.Transaction(func(tx *gorm.DB) error {
var err error
deletedAuthVersion, err = IncrementUserAuthVersionWithTx(tx, user.Id)
if err != nil {
return err
}
revokedAccessTokens, err = DeleteUserAccessTokensWithTx(tx, user.Id)
if err != nil {
return err
}
if common.RedisEnabled {
if err := tx.Unscoped().Select("id", commonKeyCol).Where("user_id = ?", user.Id).Find(&tokens).Error; err != nil {
return err
@@ -1021,18 +1021,18 @@ func (user *User) HardDelete() error {
return tx.Unscoped().Delete(user).Error
})
if err != nil {
return err
return 0, err
}
if err := publishCommittedUserAuthVersion(user.Id, deletedAuthVersion); err != nil {
common.SysError(fmt.Sprintf("failed to publish auth tombstone after hard deleting user %d: %v", user.Id, err))
common.SysError(common.LogText("failed to publish auth tombstone after hard deleting user %d: %v", user.Id, err))
}
if err := invalidateTokensCache(tokens); err != nil {
common.SysError(fmt.Sprintf("failed to invalidate token cache after hard deleting user %d: %v", user.Id, err))
common.SysError(common.LogText("failed to invalidate token cache after hard deleting user %d: %v", user.Id, err))
}
if err := invalidateUserCache(user.Id); err != nil {
common.SysError(fmt.Sprintf("failed to invalidate user cache after hard deleting user %d: %v", user.Id, err))
common.SysError(common.LogText("failed to invalidate user cache after hard deleting user %d: %v", user.Id, err))
}
return nil
return revokedAccessTokens, nil
}
func deleteUserAuthenticationData(tx *gorm.DB, userId int) error {
@@ -1084,7 +1084,7 @@ func (user *User) ValidateAndFill() (err error) {
func (user *User) FillUserById() error {
if user.Id == 0 {
return errors.New("id 为空!")
return common.NewMessage("ID is empty!")
}
DB.Where(User{Id: user.Id}).First(user)
return nil
@@ -1092,7 +1092,7 @@ func (user *User) FillUserById() error {
func (user *User) FillUserByEmail() error {
if user.Email == "" {
return errors.New("email 为空!")
return common.NewMessage("Email is empty!")
}
DB.Where(User{Email: user.Email}).First(user)
return nil
@@ -1100,7 +1100,7 @@ func (user *User) FillUserByEmail() error {
func (user *User) FillUserByGitHubId() error {
if user.GitHubId == "" {
return errors.New("GitHub id 为空!")
return common.NewMessage("GitHub ID is empty!")
}
DB.Where(User{GitHubId: user.GitHubId}).First(user)
return nil
@@ -1108,7 +1108,7 @@ func (user *User) FillUserByGitHubId() error {
func (user *User) FillUserByDiscordId() error {
if user.DiscordId == "" {
return errors.New("discord id 为空!")
return common.NewMessage("Discord ID is empty!")
}
DB.Where(User{DiscordId: user.DiscordId}).First(user)
return nil
@@ -1116,7 +1116,7 @@ func (user *User) FillUserByDiscordId() error {
func (user *User) FillUserByOidcId() error {
if user.OidcId == "" {
return errors.New("oidc id 为空!")
return common.NewMessage("OIDC ID is empty!")
}
DB.Where(User{OidcId: user.OidcId}).First(user)
return nil
@@ -1124,7 +1124,7 @@ func (user *User) FillUserByOidcId() error {
func (user *User) FillUserByWeChatId() error {
if user.WeChatId == "" {
return errors.New("WeChat id 为空!")
return common.NewMessage("WeChat ID is empty!")
}
DB.Where(User{WeChatId: user.WeChatId}).First(user)
return nil
@@ -1132,11 +1132,11 @@ func (user *User) FillUserByWeChatId() error {
func (user *User) FillUserByTelegramId() error {
if user.TelegramId == "" {
return errors.New("Telegram id 为空!")
return common.NewMessage("Telegram ID is empty!")
}
err := DB.Where(User{TelegramId: user.TelegramId}).First(user).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return errors.New("该 Telegram 账户未绑定")
return common.NewMessage("This Telegram account is not linked. Sign in using another method and link it first.")
}
return nil
}
@@ -1187,7 +1187,7 @@ func IsTelegramIdAlreadyTaken(telegramId string) bool {
func ResetUserPasswordByEmail(email string, password string) error {
if email == "" || password == "" {
return errors.New("邮箱地址或密码为空!")
return common.NewMessage("Email or password is empty!")
}
user, err := GetUniqueUserByEmail(email)
if err != nil {
@@ -1219,16 +1219,23 @@ func IsAdmin(userId int) bool {
var user User
err := DB.Where("id = ?", userId).Select("role").Find(&user).Error
if err != nil {
common.SysLog("no such user " + err.Error())
common.SysLog(common.LogText("no such user %s", err.Error()))
return false
}
return user.Role >= common.RoleAdminUser
}
// ValidateAccessToken resolves a legacy plaintext access token. After the
// transition deadline it rejects every value without querying the database.
//
// Deprecated: 旧版面板访问令牌,仅在升级后的过渡期内使用;删除 users.access_token 列时一并移除。
func ValidateAccessToken(token string) (*User, error) {
if token == "" {
return nil, nil
}
if LegacyAccessTokensRetired(common.GetTimestamp()) {
return nil, ErrLegacyAccessTokenRetired
}
token = strings.Replace(token, "Bearer ", "", 1)
user := &User{}
err := DB.Where("access_token = ?", token).First(user).Error
@@ -1271,7 +1278,7 @@ func GetUserGroup(id int, fromDB bool) (group string, err error) {
if shouldUpdateRedis(fromDB, err) {
gopool.Go(func() {
if err := RefreshUserGroupCache(id); err != nil {
common.SysLog("failed to update user group cache: " + err.Error())
common.SysLog(common.LogText("failed to update user group cache: %s", err.Error()))
}
})
}
@@ -1300,7 +1307,7 @@ func GetUserSetting(id int, fromDB bool) (settingMap dto.UserSetting, err error)
if shouldUpdateRedis(fromDB, err) {
gopool.Go(func() {
if err := updateUserSettingCache(id, setting); err != nil {
common.SysLog("failed to update user setting cache: " + err.Error())
common.SysLog(common.LogText("failed to update user setting cache: %s", err.Error()))
}
})
}
@@ -1332,7 +1339,7 @@ func GetUserSetting(id int, fromDB bool) (settingMap dto.UserSetting, err error)
func IncreaseUserQuota(id int, quota int, db bool) (err error) {
if quota < 0 {
return errors.New("quota 不能为负数!")
return errors.New("quota cannot be negative")
}
if err := common.ValidateWalletQuota(quota); err != nil {
return err
@@ -1341,7 +1348,7 @@ func IncreaseUserQuota(id int, quota int, db bool) (err error) {
addNewRecord(BatchUpdateTypeUserQuota, id, quota)
gopool.Go(func() {
if err := cacheIncrUserQuota(id, int64(quota)); err != nil {
common.SysLog("failed to increase user quota: " + err.Error())
common.SysLog(common.LogText("failed to increase user quota: %s", err.Error()))
}
})
return nil
@@ -1351,7 +1358,7 @@ func IncreaseUserQuota(id int, quota int, db bool) (err error) {
}
gopool.Go(func() {
if err := cacheIncrUserQuota(id, int64(quota)); err != nil {
common.SysLog("failed to increase user quota: " + err.Error())
common.SysLog(common.LogText("failed to increase user quota: %s", err.Error()))
}
})
return nil
@@ -1379,12 +1386,12 @@ func increaseUserQuota(id int, quota int) (err error) {
func DecreaseUserQuota(id int, quota int, db bool) (err error) {
if quota < 0 {
return errors.New("quota 不能为负数!")
return errors.New("quota cannot be negative")
}
gopool.Go(func() {
err := cacheDecrUserQuota(id, int64(quota))
if err != nil {
common.SysLog("failed to decrease user quota: " + err.Error())
common.SysLog(common.LogText("failed to decrease user quota: %s", err.Error()))
}
})
if !db && common.BatchUpdateEnabled {
@@ -1418,14 +1425,9 @@ func DeltaUpdateUserQuota(id int, delta int) (err error) {
// return email
//}
func GetRootUser() (user *User) {
DB.Where("role = ?", common.RoleRootUser).First(&user)
return user
}
func UpdateUserLastLoginAt(id int) {
if err := DB.Model(&User{}).Where("id = ?", id).Update("last_login_at", common.GetTimestamp()).Error; err != nil {
common.SysLog("failed to update user last_login_at: " + err.Error())
common.SysLog(common.LogText("failed to update user last_login_at: %s", err.Error()))
}
}
@@ -1445,7 +1447,7 @@ func UpdateUserUsedQuota(id int, quota int) {
return
}
if err := DB.Model(&User{}).Where("id = ?", id).Update("used_quota", gorm.Expr("used_quota + ?", quota)).Error; err != nil {
common.SysLog("failed to update user used quota: " + err.Error())
common.SysLog(common.LogText("failed to update user used quota: %s", err.Error()))
}
}
@@ -1457,7 +1459,7 @@ func updateUserUsedQuotaAndRequestCount(id int, quota int, count int) {
},
).Error
if err != nil {
common.SysLog("failed to update user used quota and request count: " + err.Error())
common.SysLog(common.LogText("failed to update user used quota and request count: %s", err.Error()))
return
}
@@ -1480,7 +1482,7 @@ func updateUserQuotaUsedQuotaAndRequestCount(id int, quota int, usedQuota int, r
},
).Error
if err != nil {
common.SysLog("failed to batch update user quota, used quota and request count: " + err.Error())
common.SysLog(common.LogText("failed to batch update user quota, used quota and request count: %s", err.Error()))
}
}
@@ -1491,7 +1493,7 @@ func GetUsernameById(id int, fromDB bool) (username string, err error) {
if shouldUpdateRedis(fromDB, err) {
gopool.Go(func() {
if err := updateUserNameCache(id, username); err != nil {
common.SysLog("failed to update user name cache: " + err.Error())
common.SysLog(common.LogText("failed to update user name cache: %s", err.Error()))
}
})
}
+329
View File
@@ -0,0 +1,329 @@
package model
import (
"errors"
"strconv"
"strings"
"sync/atomic"
"time"
"github.com/QuantumNous/new-api/common"
"gorm.io/gorm"
"gorm.io/gorm/clause"
)
const (
AccessTokenPrefix = "nap_"
AccessTokenSessionPrefix = "pat:"
AccessTokenSessionVersion = 1
legacyAccessTokenRetireAtKey = "LegacyAccessTokenRetireAt"
legacyAccessTokenTransitionDays = 30
userAccessTokenCleanupBatchSize = 500
userAccessTokenTouchIntervalSecs = 60
)
var (
ErrAccessTokenNotFound = errors.New("access token not found")
ErrAccessTokenLimit = errors.New("access token limit reached")
ErrAccessTokenExpired = errors.New("access token has expired")
ErrLegacyAccessTokenRetired = errors.New("legacy access token has been retired")
errLegacyRetireAtReadOnly = errors.New("LegacyAccessTokenRetireAt is managed by the server")
)
// legacyAccessTokenRetireAt caches the deployment-wide deadline loaded at
// startup. Zero means it was never loaded, which is treated as retired.
var legacyAccessTokenRetireAt atomic.Int64
// UserAccessToken is a scoped dashboard personal access token. Only the
// SHA-256 fingerprint of the plaintext is stored; it doubles as the audit
// token_ref.
type UserAccessToken struct {
Id int `json:"id"`
UserId int `json:"-" gorm:"index"`
Name string `json:"name" gorm:"type:varchar(64)"`
TokenHash string `json:"token_ref" gorm:"type:varchar(64);uniqueIndex"`
TokenHint string `json:"token_hint" gorm:"type:varchar(8)"`
Scopes string `json:"-" gorm:"type:varchar(2048)"`
ExpiresAt int64 `json:"expires_at" gorm:"type:bigint;index"`
LastUsedAt int64 `json:"last_used_at" gorm:"type:bigint"`
LastUsedIp string `json:"last_used_ip" gorm:"type:varchar(64)"`
CreatedAt int64 `json:"created_at" gorm:"type:bigint"`
}
func (UserAccessToken) TableName() string {
return "user_access_tokens"
}
// GetScopes returns the stored scope list; a malformed value grants nothing.
func (token *UserAccessToken) GetScopes() []string {
if token == nil || token.Scopes == "" {
return nil
}
var scopes []string
if err := common.UnmarshalJsonStr(token.Scopes, &scopes); err != nil {
common.SysError(common.LogText("invalid access token scopes: %s", err.Error()))
return nil
}
return scopes
}
func (token *UserAccessToken) SetScopes(scopes []string) error {
data, err := common.Marshal(scopes)
if err != nil {
return err
}
token.Scopes = string(data)
return nil
}
// Expired reports whether the token has an absolute expiry at or before now.
func (token *UserAccessToken) Expired(now int64) bool {
return token.ExpiresAt > 0 && token.ExpiresAt <= now
}
func FindUserAccessTokenByHash(hash string) (*UserAccessToken, error) {
if hash == "" {
return nil, nil
}
var token UserAccessToken
err := DB.Where("token_hash = ?", hash).First(&token).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, nil
}
if err != nil {
return nil, err
}
return &token, nil
}
func GetUserAccessToken(userID, id int) (*UserAccessToken, error) {
var token UserAccessToken
err := DB.Where("id = ? AND user_id = ?", id, userID).First(&token).Error
if errors.Is(err, gorm.ErrRecordNotFound) {
return nil, ErrAccessTokenNotFound
}
if err != nil {
return nil, err
}
return &token, nil
}
func ListUserAccessTokens(userID int) ([]UserAccessToken, error) {
tokens := make([]UserAccessToken, 0)
err := DB.Where("user_id = ?", userID).Order("created_at DESC").Order("id DESC").Find(&tokens).Error
return tokens, err
}
// CreateUserAccessToken inserts a token while holding the owner's users row, so
// concurrent creates cannot exceed limit. Expired rows awaiting cleanup count.
func CreateUserAccessToken(userID int, token *UserAccessToken, limit int) error {
if userID <= 0 || token == nil || token.TokenHash == "" {
return ErrAccessTokenNotFound
}
return DB.Transaction(func(tx *gorm.DB) error {
var user User
if err := lockForUpdate(tx).Select("id").First(&user, userID).Error; err != nil {
return err
}
var count int64
if err := tx.Model(&UserAccessToken{}).Where("user_id = ?", userID).Count(&count).Error; err != nil {
return err
}
if count >= int64(limit) {
return ErrAccessTokenLimit
}
token.Id = 0
token.UserId = userID
if token.CreatedAt == 0 {
token.CreatedAt = common.GetTimestamp()
}
return tx.Create(token).Error
})
}
// UpdateUserAccessToken renames one token owned by userID and, when scopes is
// not nil, replaces its grant.
func UpdateUserAccessToken(userID, id int, name string, scopes []string) (*UserAccessToken, error) {
updates := map[string]any{"name": name}
if scopes != nil {
var grant UserAccessToken
if err := grant.SetScopes(scopes); err != nil {
return nil, err
}
updates["scopes"] = grant.Scopes
}
result := DB.Model(&UserAccessToken{}).Where("id = ? AND user_id = ?", id, userID).Updates(updates)
if result.Error != nil {
return nil, result.Error
}
token, err := GetUserAccessToken(userID, id)
if err != nil {
return nil, err
}
return token, nil
}
// DeleteUserAccessToken hard-deletes one token owned by userID and returns the
// deleted row for auditing.
func DeleteUserAccessToken(userID, id int) (*UserAccessToken, error) {
var token UserAccessToken
err := DB.Transaction(func(tx *gorm.DB) error {
if err := lockForUpdate(tx).Where("id = ? AND user_id = ?", id, userID).First(&token).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return ErrAccessTokenNotFound
}
return err
}
return tx.Where("id = ? AND user_id = ?", id, userID).Delete(&UserAccessToken{}).Error
})
if err != nil {
return nil, err
}
return &token, nil
}
func DeleteUserAccessTokensWithTx(tx *gorm.DB, userID int) (int64, error) {
result := tx.Where("user_id = ?", userID).Delete(&UserAccessToken{})
return result.RowsAffected, result.Error
}
// TouchUserAccessToken records the last use at most once per interval per token.
func TouchUserAccessToken(id int, ip string, now int64) error {
if len(ip) > 64 {
ip = ip[:64]
}
return DB.Model(&UserAccessToken{}).
Where("id = ? AND last_used_at < ?", id, now-userAccessTokenTouchIntervalSecs).
Updates(map[string]any{"last_used_at": now, "last_used_ip": ip}).Error
}
// DeleteExpiredUserAccessTokens removes tokens that expired before the cutoff.
// Tokens that never expire are kept.
func DeleteExpiredUserAccessTokens(before int64) error {
for {
var ids []int
if err := DB.Model(&UserAccessToken{}).
Where("expires_at > 0 AND expires_at < ?", before).
Order("id").Limit(userAccessTokenCleanupBatchSize).Pluck("id", &ids).Error; err != nil {
return err
}
if len(ids) == 0 {
return nil
}
if err := DB.Where("id IN ?", ids).Delete(&UserAccessToken{}).Error; err != nil {
return err
}
if len(ids) < userAccessTokenCleanupBatchSize {
return nil
}
}
}
// AccessTokenHint is the last four characters of a token, shown so users can
// tell their tokens apart. PostgreSQL pads legacy CHAR(32) values with spaces.
func AccessTokenHint(token string) string {
token = strings.TrimRight(token, " ")
if len(token) <= 4 {
return ""
}
return token[len(token)-4:]
}
// AccessTokenSessionID is the step-up session identifier bound to a token.
// Browser session IDs are UUIDs and can never carry this prefix.
func AccessTokenSessionID(id int) string {
return AccessTokenSessionPrefix + strconv.Itoa(id)
}
func ParseAccessTokenSessionID(sid string) (int, bool) {
raw, ok := strings.CutPrefix(sid, AccessTokenSessionPrefix)
if !ok {
return 0, false
}
id, err := strconv.Atoi(raw)
if err != nil || id <= 0 || strconv.Itoa(id) != raw {
return 0, false
}
return id, true
}
// ValidateAccessTokenIdentity checks a token-bound step-up identity without
// taking locks, for use before issuing a verification proof.
func ValidateAccessTokenIdentity(identity AuthSessionIdentity) error {
tokenID, ok := ParseAccessTokenSessionID(identity.SessionID)
if !ok {
return ErrUserSessionInactive
}
return checkAccessTokenIdentity(DB, tokenID, identity, false)
}
func validateAccessTokenIdentityWithTx(tx *gorm.DB, tokenID int, identity AuthSessionIdentity) error {
return checkAccessTokenIdentity(tx, tokenID, identity, true)
}
// checkAccessTokenIdentity locks users before user_access_tokens, matching
// CreateUserAccessToken, and reports every mismatch as an inactive session.
func checkAccessTokenIdentity(tx *gorm.DB, tokenID int, identity AuthSessionIdentity, lock bool) error {
if identity.UserID <= 0 || identity.UserAuthVersion <= 0 || identity.SessionVersion != AccessTokenSessionVersion {
return ErrUserSessionInactive
}
query := tx
if lock {
query = lockForUpdate(tx)
}
var user User
if err := query.Select("id", "status", "auth_version").First(&user, identity.UserID).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return ErrUserSessionInactive
}
return err
}
if user.Status != common.UserStatusEnabled || user.AuthVersion != identity.UserAuthVersion {
return ErrUserSessionInactive
}
query = tx
if lock {
query = lockForUpdate(tx)
}
var token UserAccessToken
if err := query.Where("id = ? AND user_id = ?", tokenID, identity.UserID).First(&token).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
return ErrUserSessionInactive
}
return err
}
if token.Expired(time.Now().Unix()) {
return ErrUserSessionInactive
}
return nil
}
// EnsureLegacyAccessTokenRetireAt records the legacy token deadline on first
// start (now + 30 days) and loads the stored value. The first node to start
// wins; later starts and other nodes only read it.
func EnsureLegacyAccessTokenRetireAt(now int64) error {
deadline := strconv.FormatInt(now+legacyAccessTokenTransitionDays*24*60*60, 10)
if err := DB.Clauses(clause.OnConflict{DoNothing: true}).Create(&Option{Key: legacyAccessTokenRetireAtKey, Value: deadline}).Error; err != nil {
return err
}
var option Option
if err := DB.Where(&Option{Key: legacyAccessTokenRetireAtKey}).First(&option).Error; err != nil {
return err
}
retireAt, err := strconv.ParseInt(strings.TrimSpace(option.Value), 10, 64)
if err != nil || retireAt <= 0 {
return errors.New("invalid LegacyAccessTokenRetireAt option value")
}
legacyAccessTokenRetireAt.Store(retireAt)
return nil
}
func LegacyAccessTokenRetireAt() int64 {
return legacyAccessTokenRetireAt.Load()
}
func LegacyAccessTokensRetired(now int64) bool {
retireAt := legacyAccessTokenRetireAt.Load()
return retireAt <= 0 || now >= retireAt
}
+9 -2
View File
@@ -28,6 +28,7 @@ func TestHardDeleteUserFailsClosedWhenAuthFenceCannotPublish(t *testing.T) {
return ClaimExternalIdentityWithTx(tx, ExternalIdentityProviderTelegram, user.TelegramId, user.Id)
}))
require.NoError(t, DB.Create(&Token{UserId: user.Id, Key: "hard-delete-token"}).Error)
require.NoError(t, DB.Create(&UserAccessToken{UserId: user.Id, TokenHash: AccessTokenFingerprint("nap_hard-delete-token")}).Error)
require.NoError(t, DB.Create(&TwoFA{UserId: user.Id, Secret: "secret", IsEnabled: true}).Error)
require.NoError(t, DB.Create(&TwoFABackupCode{UserId: user.Id, CodeHash: "hash"}).Error)
require.NoError(t, DB.Create(&PasskeyCredential{UserID: user.Id, CredentialID: "credential", PublicKey: "public-key"}).Error)
@@ -55,7 +56,8 @@ func TestHardDeleteUserFailsClosedWhenAuthFenceCannotPublish(t *testing.T) {
common.RedisEnabled, common.RDB = oldRedisEnabled, oldRDB
})
require.Error(t, HardDeleteUserById(user.Id))
_, err := HardDeleteUserById(user.Id)
require.Error(t, err)
var count int64
require.NoError(t, DB.Unscoped().Model(&User{}).Where("id = ?", user.Id).Count(&count).Error)
@@ -69,6 +71,7 @@ func TestHardDeleteUserFailsClosedWhenAuthFenceCannotPublish(t *testing.T) {
&UserSession{},
&AuthFlow{},
&ExternalIdentityClaim{},
&UserAccessToken{},
} {
require.NoError(t, DB.Unscoped().Model(record).Where("user_id = ?", user.Id).Count(&count).Error)
assert.EqualValues(t, 1, count)
@@ -88,6 +91,7 @@ func TestHardDeleteUserPublishesTombstoneAndPurgesAuthenticationData(t *testing.
return ClaimExternalIdentityWithTx(tx, ExternalIdentityProviderTelegram, user.TelegramId, user.Id)
}))
require.NoError(t, DB.Create(&Token{UserId: user.Id, Key: "hard-delete-success-token"}).Error)
require.NoError(t, DB.Create(&UserAccessToken{UserId: user.Id, TokenHash: AccessTokenFingerprint("nap_hard-delete-success-token")}).Error)
require.NoError(t, DB.Create(&TwoFA{UserId: user.Id, Secret: "secret", IsEnabled: true}).Error)
require.NoError(t, DB.Create(&TwoFABackupCode{UserId: user.Id, CodeHash: "hash"}).Error)
require.NoError(t, DB.Create(&PasskeyCredential{UserID: user.Id, CredentialID: "credential-success", PublicKey: "public-key"}).Error)
@@ -106,7 +110,9 @@ func TestHardDeleteUserPublishesTombstoneAndPurgesAuthenticationData(t *testing.
// user; the shared version increment must therefore query unscoped.
require.NoError(t, DB.Delete(&user).Error)
require.NoError(t, HardDeleteUserById(user.Id))
revokedAccessTokens, err := HardDeleteUserById(user.Id)
require.NoError(t, err)
assert.EqualValues(t, 1, revokedAccessTokens)
var count int64
require.NoError(t, DB.Unscoped().Model(&User{}).Where("id = ?", user.Id).Count(&count).Error)
@@ -120,6 +126,7 @@ func TestHardDeleteUserPublishesTombstoneAndPurgesAuthenticationData(t *testing.
&UserSession{},
&AuthFlow{},
&ExternalIdentityClaim{},
&UserAccessToken{},
} {
require.NoError(t, DB.Unscoped().Model(record).Where("user_id = ?", user.Id).Count(&count).Error)
assert.Zero(t, count)
+3 -3
View File
@@ -40,7 +40,7 @@ func (user *UserBase) GetSetting() dto.UserSetting {
if user.Setting != "" {
err := common.Unmarshal([]byte(user.Setting), &setting)
if err != nil {
common.SysLog("failed to unmarshal setting: " + err.Error())
common.SysLog(common.LogText("failed to unmarshal setting: %s", err.Error()))
}
}
return setting
@@ -108,7 +108,7 @@ func GetUserCache(userId int) (*UserBase, error) {
if errors.Is(err, ErrUserAuthCachePending) {
return nil, err
}
common.SysLog("failed to synchronously populate user cache: " + err.Error())
common.SysLog(common.LogText("failed to synchronously populate user cache: %s", err.Error()))
}
}
return user.ToBaseUser(), nil
@@ -160,7 +160,7 @@ func syncCreditUserQuotaCache(userId int, quota int, operation string) {
return
}
if err := cacheIncrUserQuota(userId, int64(quota)); err != nil {
common.SysLog(fmt.Sprintf("failed to sync %s credit to user quota cache: %s", operation, err.Error()))
common.SysLog(common.LogText("failed to sync %s credit to user quota cache: %s", operation, err.Error()))
}
}
+1 -2
View File
@@ -2,7 +2,6 @@ package model
import (
"errors"
"fmt"
"github.com/QuantumNous/new-api/common"
"github.com/shopspring/decimal"
@@ -80,7 +79,7 @@ func AdjustUserQuota(userID, operatorRole int, mode string, value int) (*UserQuo
delta := int64(adjustment.After) - int64(adjustment.Before)
if delta != 0 {
if err := cacheIncrUserQuota(userID, delta); err != nil {
common.SysError(fmt.Sprintf("failed to sync manual quota adjustment for user %d: %s", userID, err))
common.SysError(common.LogText("failed to sync manual quota adjustment for user %d: %s", userID, err))
}
}
return &adjustment, nil
+7 -7
View File
@@ -169,7 +169,7 @@ func publishCreatedUserSession(session *UserSession, cacheDeadline time.Time) er
if errors.Is(err, ErrUserSessionInactive) {
return err
}
common.SysLog("failed to populate newly created user session cache: " + err.Error())
common.SysLog(common.LogText("failed to populate newly created user session cache: %s", err.Error()))
}
return nil
}
@@ -255,7 +255,7 @@ func GetUserSessionCached(sid string) (*UserSession, error) {
if errors.Is(err, ErrUserSessionInactive) {
return nil, err
}
common.SysLog("failed to synchronously populate user session cache: " + err.Error())
common.SysLog(common.LogText("failed to synchronously populate user session cache: %s", err.Error()))
}
}
return session, nil
@@ -505,7 +505,7 @@ func RotateUserSessionRefresh(userID int, sid, presentedHash, nextHash string, n
} else if errors.Is(err, ErrUserSessionInactive) {
return nil, err
} else {
common.SysLog("failed to update rotated user session cache: " + err.Error())
common.SysLog(common.LogText("failed to update rotated user session cache: %s", err.Error()))
}
}
return &session, nil
@@ -542,7 +542,7 @@ func RotateUserSessionRefresh(userID int, sid, presentedHash, nextHash string, n
session.RevokedAt = now
session.RevokedReason = "refresh_reuse"
if err := writeUserSessionCache(session.cacheEntry(), time.Time{}); err != nil {
common.SysLog("failed to cache refresh-reuse session revoke: " + err.Error())
common.SysLog(common.LogText("failed to cache refresh-reuse session revoke: %s", err.Error()))
}
return nil, ErrUserSessionRefreshReuse
}
@@ -596,7 +596,7 @@ func RevokeUserSession(userID int, sid, reason string) (bool, error) {
candidate.RevokedAt = now
candidate.RevokedReason = reason
if err := writeUserSessionCache(candidate.cacheEntry(), time.Time{}); err != nil {
common.SysLog("failed to finalize user session revoke tombstone: " + err.Error())
common.SysLog(common.LogText("failed to finalize user session revoke tombstone: %s", err.Error()))
}
}
return revoked, nil
@@ -652,7 +652,7 @@ func RevokeUserSessionByRefreshHash(sid, presentedHash, reason string) (bool, er
}
if revoked {
if err := writeUserSessionCache(session.cacheEntry(), time.Time{}); err != nil {
common.SysLog("failed to finalize refresh-authenticated session revoke tombstone: " + err.Error())
common.SysLog(common.LogText("failed to finalize refresh-authenticated session revoke tombstone: %s", err.Error()))
}
}
return revoked, nil
@@ -775,7 +775,7 @@ func revokeUserSessions(userID int, excludedSID, reason string) (int64, error) {
revoked[i].RevokedAt = now
revoked[i].RevokedReason = reason
if err := writeUserSessionCache(revoked[i].cacheEntry(), time.Time{}); err != nil {
common.SysLog("failed to finalize bulk user session revoke tombstone: " + err.Error())
common.SysLog(common.LogText("failed to finalize bulk user session revoke tombstone: %s", err.Error()))
}
}
}
+239 -14
View File
@@ -2,7 +2,9 @@ package model
import (
"errors"
"strconv"
"testing"
"time"
"github.com/QuantumNous/new-api/common"
"github.com/QuantumNous/new-api/relaykit/dto"
@@ -144,19 +146,22 @@ func TestUsageAccountingSupportsSignedDirectAndBatchDeltas(t *testing.T) {
assert.Equal(t, int64(1150), gotChannel.UsedQuota)
}
func TestUpdateUserAccessTokenOnlyUpdatesAccessToken(t *testing.T) {
func TestRevokeUserAccessTokenOnlyClearsLegacyToken(t *testing.T) {
setupUserUpdateTestState(t)
createdAt := int64(100)
user := User{
Id: 2,
Username: "token-rotation-user",
Password: "password",
DisplayName: "before",
Status: common.UserStatusEnabled,
Quota: 1000,
AffQuota: 800,
AffHistoryQuota: 1200,
Id: 2,
Username: "token-revoke-user",
Password: "password",
DisplayName: "before",
Status: common.UserStatusEnabled,
Quota: 1000,
AffQuota: 800,
AffHistoryQuota: 1200,
AccessTokenCreatedAt: &createdAt,
}
user.SetAccessToken("legacy-token")
require.NoError(t, DB.Create(&user).Error)
require.NoError(t, DB.Model(&User{}).Where("id = ?", user.Id).Updates(map[string]any{
@@ -165,23 +170,26 @@ func TestUpdateUserAccessTokenOnlyUpdatesAccessToken(t *testing.T) {
"display_name": "concurrent-update",
}).Error)
require.NoError(t, UpdateUserAccessToken(user.Id, "rotated-token"))
tokenRef, err := RevokeUserAccessToken(user.Id)
require.NoError(t, err)
assert.Equal(t, AccessTokenFingerprint("legacy-token"), tokenRef)
var got User
require.NoError(t, DB.First(&got, user.Id).Error)
assert.Equal(t, "rotated-token", got.GetAccessToken())
assert.Empty(t, got.GetAccessToken())
assert.Nil(t, got.AccessTokenCreatedAt)
assert.Equal(t, "concurrent-update", got.DisplayName)
assert.Equal(t, 1500, got.Quota)
assert.Equal(t, 300, got.AffQuota)
assert.Equal(t, 1200, got.AffHistoryQuota)
}
func TestUpdateUserAccessTokenRejectsSoftDeletedUser(t *testing.T) {
func TestRevokeUserAccessTokenRejectsSoftDeletedUser(t *testing.T) {
setupUserUpdateTestState(t)
user := User{
Id: 3,
Username: "deleted-token-rotation-user",
Username: "deleted-token-revoke-user",
Password: "password",
Status: common.UserStatusEnabled,
}
@@ -189,7 +197,7 @@ func TestUpdateUserAccessTokenRejectsSoftDeletedUser(t *testing.T) {
require.NoError(t, DB.Create(&user).Error)
require.NoError(t, DB.Delete(&user).Error)
err := UpdateUserAccessToken(user.Id, "orphaned-token")
_, err := RevokeUserAccessToken(user.Id)
require.ErrorIs(t, err, gorm.ErrRecordNotFound)
var got User
@@ -398,3 +406,220 @@ func TestResetUserPasswordByEmailRequiresSingleActiveMatch(t *testing.T) {
err = ResetUserPasswordByEmail("missing@example.com", "NewPassword123")
require.True(t, errors.Is(err, ErrEmailNotFound))
}
func createAccessTokenTestUser(t *testing.T, username string) User {
t.Helper()
user := User{
Username: username,
Password: "unused-password-hash",
Role: common.RoleCommonUser,
Status: common.UserStatusEnabled,
Group: "default",
AuthVersion: 1,
AffCode: username + "-aff",
}
require.NoError(t, DB.Create(&user).Error)
return user
}
func createTestUserAccessToken(t *testing.T, userID int, raw string, expiresAt int64) UserAccessToken {
t.Helper()
token := UserAccessToken{Name: raw, TokenHash: AccessTokenFingerprint(raw), TokenHint: AccessTokenHint(raw), ExpiresAt: expiresAt}
require.NoError(t, token.SetScopes([]string{"profile:read"}))
require.NoError(t, CreateUserAccessToken(userID, &token, 20))
return token
}
func TestLegacyAccessTokenRetireAtIsWrittenOnceAndReadOnly(t *testing.T) {
require.NoError(t, DB.AutoMigrate(&Option{}))
previous := legacyAccessTokenRetireAt.Load()
originalMap := common.OptionMap
common.OptionMap = map[string]string{}
require.NoError(t, DB.Where(&Option{Key: legacyAccessTokenRetireAtKey}).Delete(&Option{}).Error)
t.Cleanup(func() {
legacyAccessTokenRetireAt.Store(previous)
common.OptionMap = originalMap
DB.Where(&Option{Key: legacyAccessTokenRetireAtKey}).Delete(&Option{})
})
const firstStart = int64(1_800_000_000)
want := firstStart + 30*24*60*60
require.NoError(t, EnsureLegacyAccessTokenRetireAt(firstStart))
assert.Equal(t, want, LegacyAccessTokenRetireAt())
require.NoError(t, EnsureLegacyAccessTokenRetireAt(firstStart+24*60*60))
assert.Equal(t, want, LegacyAccessTokenRetireAt())
var stored Option
require.NoError(t, DB.Where(&Option{Key: legacyAccessTokenRetireAtKey}).First(&stored).Error)
assert.Equal(t, strconv.FormatInt(want, 10), stored.Value)
require.ErrorIs(t, UpdateOption(legacyAccessTokenRetireAtKey, "1"), errLegacyRetireAtReadOnly)
require.ErrorIs(t, UpdateOptionsBulk(map[string]string{legacyAccessTokenRetireAtKey: "1"}), errLegacyRetireAtReadOnly)
require.NoError(t, DB.Where(&Option{Key: legacyAccessTokenRetireAtKey}).First(&stored).Error)
assert.Equal(t, strconv.FormatInt(want, 10), stored.Value)
require.NoError(t, updateOptionMap(legacyAccessTokenRetireAtKey, stored.Value))
assert.NotContains(t, common.OptionMap, legacyAccessTokenRetireAtKey)
}
func TestValidateAccessTokenStopsAtLegacyDeadline(t *testing.T) {
setupUserUpdateTestState(t)
previous := legacyAccessTokenRetireAt.Load()
t.Cleanup(func() { legacyAccessTokenRetireAt.Store(previous) })
user := createAccessTokenTestUser(t, "legacy-deadline-user")
require.NoError(t, DB.Model(&User{}).Where("id = ?", user.Id).Update("access_token", "legacy-deadline-token").Error)
legacyAccessTokenRetireAt.Store(time.Now().Add(time.Hour).Unix())
found, err := ValidateAccessToken("legacy-deadline-token")
require.NoError(t, err)
require.NotNil(t, found)
assert.Equal(t, user.Id, found.Id)
var userQueries int
const callbackName = "test:count_legacy_token_queries"
require.NoError(t, DB.Callback().Query().Before("gorm:query").Register(callbackName, func(tx *gorm.DB) {
if tx.Statement.Table == "users" {
userQueries++
}
}))
t.Cleanup(func() { _ = DB.Callback().Query().Remove(callbackName) })
for _, retireAt := range []int64{time.Now().Add(-time.Second).Unix(), 0} {
legacyAccessTokenRetireAt.Store(retireAt)
found, err = ValidateAccessToken("legacy-deadline-token")
require.ErrorIs(t, err, ErrLegacyAccessTokenRetired)
assert.Nil(t, found)
}
assert.Zero(t, userQueries)
}
func TestUserAccessTokenLimitAndOwnership(t *testing.T) {
setupUserUpdateTestState(t)
owner := createAccessTokenTestUser(t, "token-owner")
other := createAccessTokenTestUser(t, "token-other")
var first UserAccessToken
for i := range 20 {
token := createTestUserAccessToken(t, owner.Id, "nap_limit_"+strconv.Itoa(i), 0)
if i == 0 {
first = token
}
}
extra := UserAccessToken{TokenHash: AccessTokenFingerprint("nap_limit_extra")}
require.ErrorIs(t, CreateUserAccessToken(owner.Id, &extra, 20), ErrAccessTokenLimit)
var count int64
require.NoError(t, DB.Model(&UserAccessToken{}).Where("user_id = ?", owner.Id).Count(&count).Error)
assert.EqualValues(t, 20, count)
_, err := UpdateUserAccessToken(other.Id, first.Id, "stolen", []string{"profile:read"})
require.ErrorIs(t, err, ErrAccessTokenNotFound)
_, err = DeleteUserAccessToken(other.Id, first.Id)
require.ErrorIs(t, err, ErrAccessTokenNotFound)
stored, err := GetUserAccessToken(owner.Id, first.Id)
require.NoError(t, err)
assert.Equal(t, first.Name, stored.Name)
assert.Equal(t, first.Scopes, stored.Scopes)
renamed, err := UpdateUserAccessToken(owner.Id, first.Id, "renamed", nil)
require.NoError(t, err)
assert.Equal(t, "renamed", renamed.Name)
assert.Equal(t, first.Scopes, renamed.Scopes, "a rename keeps the grant")
regranted, err := UpdateUserAccessToken(owner.Id, first.Id, "regranted", []string{"usage:read"})
require.NoError(t, err)
assert.Equal(t, "regranted", regranted.Name)
assert.Equal(t, []string{"usage:read"}, regranted.GetScopes())
deleted, err := DeleteUserAccessToken(owner.Id, first.Id)
require.NoError(t, err)
assert.Equal(t, first.TokenHash, deleted.TokenHash)
found, err := FindUserAccessTokenByHash(first.TokenHash)
require.NoError(t, err)
assert.Nil(t, found)
}
func TestDisablingUserKeepsAccessTokensAndDeletingRemovesThem(t *testing.T) {
setupUserUpdateTestState(t)
user := createAccessTokenTestUser(t, "token-lifecycle-user")
createTestUserAccessToken(t, user.Id, "nap_lifecycle_one", 0)
createTestUserAccessToken(t, user.Id, "nap_lifecycle_two", time.Now().Add(time.Hour).Unix())
countTokens := func() int64 {
var count int64
require.NoError(t, DB.Model(&UserAccessToken{}).Where("user_id = ?", user.Id).Count(&count).Error)
return count
}
user.Status = common.UserStatusDisabled
require.NoError(t, user.Update(false))
assert.EqualValues(t, 2, countTokens())
revoked, err := user.Delete()
require.NoError(t, err)
assert.EqualValues(t, 2, revoked)
assert.Zero(t, countTokens())
hardDeleted := createAccessTokenTestUser(t, "token-hard-delete-user")
createTestUserAccessToken(t, hardDeleted.Id, "nap_hard_delete", 0)
revoked, err = hardDeleted.HardDelete()
require.NoError(t, err)
assert.EqualValues(t, 1, revoked)
var remaining int64
require.NoError(t, DB.Model(&UserAccessToken{}).Where("user_id = ?", hardDeleted.Id).Count(&remaining).Error)
assert.Zero(t, remaining)
}
func TestValidateAuthSessionWithTxChecksAccessTokenIdentity(t *testing.T) {
setupUserUpdateTestState(t)
now := time.Now()
tests := []struct {
name string
prepare func(user *User, token *UserAccessToken, identity *AuthSessionIdentity)
wantErr bool
}{
{name: "active token", prepare: func(*User, *UserAccessToken, *AuthSessionIdentity) {}},
{name: "expired token", wantErr: true, prepare: func(_ *User, token *UserAccessToken, _ *AuthSessionIdentity) {
require.NoError(t, DB.Model(token).Update("expires_at", now.Add(-time.Second).Unix()).Error)
}},
{name: "deleted token", wantErr: true, prepare: func(user *User, token *UserAccessToken, _ *AuthSessionIdentity) {
_, err := DeleteUserAccessToken(user.Id, token.Id)
require.NoError(t, err)
}},
{name: "another user's token", wantErr: true, prepare: func(_ *User, _ *UserAccessToken, identity *AuthSessionIdentity) {
other := createAccessTokenTestUser(t, "token-identity-other")
identity.UserID = other.Id
}},
{name: "stale auth version", wantErr: true, prepare: func(user *User, _ *UserAccessToken, _ *AuthSessionIdentity) {
require.NoError(t, DB.Model(user).Update("auth_version", 2).Error)
}},
{name: "disabled user", wantErr: true, prepare: func(user *User, _ *UserAccessToken, _ *AuthSessionIdentity) {
require.NoError(t, DB.Model(user).Update("status", common.UserStatusDisabled).Error)
}},
{name: "unexpected session version", wantErr: true, prepare: func(_ *User, _ *UserAccessToken, identity *AuthSessionIdentity) {
identity.SessionVersion = 2
}},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
require.NoError(t, DB.Exec("DELETE FROM user_access_tokens").Error)
require.NoError(t, DB.Exec("DELETE FROM users").Error)
user := createAccessTokenTestUser(t, "token-identity-user")
token := createTestUserAccessToken(t, user.Id, "nap_identity_token", now.Add(time.Hour).Unix())
identity := AuthSessionIdentity{
UserID: user.Id, SessionID: AccessTokenSessionID(token.Id),
UserAuthVersion: 1, SessionVersion: AccessTokenSessionVersion,
}
tt.prepare(&user, &token, &identity)
txErr := DB.Transaction(func(tx *gorm.DB) error { return ValidateAuthSessionWithTx(tx, identity) })
plainErr := ValidateAccessTokenIdentity(identity)
if !tt.wantErr {
assert.NoError(t, txErr)
assert.NoError(t, plainErr)
return
}
assert.ErrorIs(t, txErr, ErrUserSessionInactive)
assert.ErrorIs(t, plainErr, ErrUserSessionInactive)
})
}
}
+4 -5
View File
@@ -2,7 +2,6 @@ package model
import (
"errors"
"fmt"
"math"
"sync"
"time"
@@ -52,7 +51,7 @@ func addNewRecord(type_ int, id int, value int) {
sum := old + value
if (value > 0 && sum < old) || (value < 0 && sum > old) {
common.SysError(fmt.Sprintf("batch update overflow: type=%d id=%d old=%d value=%d", type_, id, old, value))
common.SysError(common.LogText("batch update overflow: type=%d id=%d old=%d value=%d", type_, id, old, value))
if value > 0 {
sum = math.MaxInt
} else {
@@ -79,7 +78,7 @@ func batchUpdate() {
return
}
common.SysLog("batch update started")
common.SysLog(common.LogText("batch update started"))
stores := make([]map[int]int, BatchUpdateTypeCount)
for i := range BatchUpdateTypeCount {
batchUpdateLocks[i].Lock()
@@ -97,7 +96,7 @@ func batchUpdate() {
case BatchUpdateTypeTokenQuota:
err := increaseTokenQuota(key, value)
if err != nil {
common.SysLog("failed to batch update token quota: " + err.Error())
common.SysLog(common.LogText("failed to batch update token quota: %s", err.Error()))
}
case BatchUpdateTypeChannelUsedQuota:
updateChannelUsedQuota(key, value)
@@ -122,7 +121,7 @@ func batchUpdate() {
for key := range userIDs {
updateUserQuotaUsedQuotaAndRequestCount(key, userQuotaStore[key], usedQuotaStore[key], requestCountStore[key])
}
common.SysLog("batch update finished")
common.SysLog(common.LogText("batch update finished"))
}
func RecordExist(err error) (bool, error) {