feat: sync deployed v1.0.0-rc.38-custom.42
Sync production source including routing, billing, Astra detection and public uptime. Update custom build instructions and license packaging.
This commit is contained in:
cneicy committed
1 parent
aa6c3f50f4
commit
ceddfd3d6b
828 files changed
+122253
-13702
No files matched your search
+63
-24
@@ -57,27 +57,29 @@ func sanitizeClickHouseLikePattern(input string) (string, error) {
|
||||
}
|
||||
|
||||
type Log struct {
|
||||
Id int `json:"id" gorm:"index:idx_created_at_id,priority:2;index:idx_user_id_id,priority:2"`
|
||||
UserId int `json:"user_id" gorm:"index;index:idx_user_id_id,priority:1"`
|
||||
CreatedAt int64 `json:"created_at" gorm:"bigint;index:idx_created_at_id,priority:1;index:idx_created_at_type"`
|
||||
Type int `json:"type" gorm:"index:idx_created_at_type"`
|
||||
Content string `json:"content"`
|
||||
Username string `json:"username" gorm:"index;index:index_username_model_name,priority:2;default:''"`
|
||||
TokenName string `json:"token_name" gorm:"index;default:''"`
|
||||
ModelName string `json:"model_name" gorm:"index;index:index_username_model_name,priority:1;default:''"`
|
||||
Quota int `json:"quota" gorm:"default:0"`
|
||||
PromptTokens int `json:"prompt_tokens" gorm:"default:0"`
|
||||
CompletionTokens int `json:"completion_tokens" gorm:"default:0"`
|
||||
UseTime int `json:"use_time" gorm:"default:0"`
|
||||
IsStream bool `json:"is_stream"`
|
||||
ChannelId int `json:"channel" gorm:"index"`
|
||||
ChannelName string `json:"channel_name" gorm:"->"`
|
||||
TokenId int `json:"token_id" gorm:"default:0;index"`
|
||||
Group string `json:"group" gorm:"index"`
|
||||
Ip string `json:"ip" gorm:"index;default:''"`
|
||||
RequestId string `json:"request_id,omitempty" gorm:"type:varchar(64);index:idx_logs_request_id;default:''"`
|
||||
UpstreamRequestId string `json:"upstream_request_id,omitempty" gorm:"type:varchar(128);index:idx_logs_upstream_request_id;default:''"`
|
||||
Other string `json:"other"`
|
||||
AttemptKey *string `json:"attempt_key,omitempty" gorm:"type:varchar(96);uniqueIndex"`
|
||||
Id int `json:"id" gorm:"index:idx_created_at_id,priority:2;index:idx_user_id_id,priority:2"`
|
||||
UserId int `json:"user_id" gorm:"index;index:idx_user_id_id,priority:1"`
|
||||
CreatedAt int64 `json:"created_at" gorm:"bigint;index:idx_created_at_id,priority:1;index:idx_created_at_type"`
|
||||
Type int `json:"type" gorm:"index:idx_created_at_type"`
|
||||
Content string `json:"content"`
|
||||
Username string `json:"username" gorm:"index;index:index_username_model_name,priority:2;default:''"`
|
||||
TokenName string `json:"token_name" gorm:"index;default:''"`
|
||||
ModelName string `json:"model_name" gorm:"index;index:index_username_model_name,priority:1;default:''"`
|
||||
Quota int `json:"quota" gorm:"default:0"`
|
||||
PromptTokens int `json:"prompt_tokens" gorm:"default:0"`
|
||||
CompletionTokens int `json:"completion_tokens" gorm:"default:0"`
|
||||
UseTime int `json:"use_time" gorm:"default:0"`
|
||||
IsStream bool `json:"is_stream"`
|
||||
ChannelId int `json:"channel" gorm:"index"`
|
||||
ChannelName string `json:"channel_name" gorm:"->"`
|
||||
TokenId int `json:"token_id" gorm:"default:0;index"`
|
||||
Group string `json:"group" gorm:"index"`
|
||||
Ip string `json:"ip" gorm:"index;default:''"`
|
||||
RequestId string `json:"request_id,omitempty" gorm:"type:varchar(64);index:idx_logs_request_id;default:''"`
|
||||
UpstreamRequestId string `json:"upstream_request_id,omitempty" gorm:"type:varchar(128);index:idx_logs_upstream_request_id;default:''"`
|
||||
CorrelationKey string `json:"-" gorm:"type:varchar(64);index:idx_logs_correlation;default:''"`
|
||||
Other string `json:"other"`
|
||||
}
|
||||
|
||||
// don't use iota, avoid change log type value
|
||||
@@ -116,6 +118,7 @@ func assignDisplayLogIds(logs []*Log, startIdx int) {
|
||||
func formatUserLogs(logs []*Log, startIdx int) {
|
||||
for i := range logs {
|
||||
logs[i].ChannelName = ""
|
||||
normalizeErrorLogGroup(logs[i])
|
||||
logs[i].Other = formatLogOtherJSON(logs[i].Other, logOtherVisibilityUser)
|
||||
}
|
||||
assignDisplayLogIds(logs, startIdx)
|
||||
@@ -125,6 +128,7 @@ func formatUserLogs(logs []*Log, startIdx int) {
|
||||
// admin_info. Root callers must not pass their results through this formatter.
|
||||
func FormatAdminLogs(logs []*Log) {
|
||||
for i := range logs {
|
||||
normalizeErrorLogGroup(logs[i])
|
||||
logs[i].Other = formatLogOtherJSON(logs[i].Other, logOtherVisibilityAdmin)
|
||||
}
|
||||
}
|
||||
@@ -133,6 +137,7 @@ func FormatAdminLogs(logs []*Log) {
|
||||
// without removing root-only diagnostics.
|
||||
func FormatRootLogs(logs []*Log) {
|
||||
for i := range logs {
|
||||
normalizeErrorLogGroup(logs[i])
|
||||
logs[i].Other = formatLogOtherJSON(logs[i].Other, logOtherVisibilityRoot)
|
||||
}
|
||||
}
|
||||
@@ -313,8 +318,10 @@ func RecordErrorLog(c *gin.Context, userId int, channelId int, modelName string,
|
||||
}(),
|
||||
RequestId: requestId,
|
||||
UpstreamRequestId: upstreamRequestId,
|
||||
CorrelationKey: c.GetString("log_correlation_key"),
|
||||
Other: otherStr,
|
||||
}
|
||||
normalizeErrorLogGroup(log)
|
||||
err := createLog(log)
|
||||
if err != nil {
|
||||
logger.LogError(c, "failed to record log: "+err.Error())
|
||||
@@ -377,6 +384,7 @@ func RecordConsumeLog(c *gin.Context, userId int, params RecordConsumeLogParams)
|
||||
}(),
|
||||
RequestId: requestId,
|
||||
UpstreamRequestId: upstreamRequestId,
|
||||
CorrelationKey: c.GetString("log_correlation_key"),
|
||||
Other: otherStr,
|
||||
}
|
||||
err := createLog(log)
|
||||
@@ -461,7 +469,7 @@ func RecordTaskBillingLog(params RecordTaskBillingLogParams) {
|
||||
}
|
||||
}
|
||||
|
||||
func GetAllLogs(logType int, startTimestamp int64, endTimestamp int64, modelName string, username string, tokenName string, startIdx int, num int, channel int, group string, requestId string, upstreamRequestId string) (logs []*Log, total int64, err error) {
|
||||
func GetAllLogs(logType int, startTimestamp int64, endTimestamp int64, modelName string, username string, tokenName string, startIdx int, num int, channel int, group string, requestId string, upstreamRequestId string, correlationKey ...string) (logs []*Log, total int64, err error) {
|
||||
var tx *gorm.DB
|
||||
if logType == LogTypeUnknown {
|
||||
tx = LOG_DB
|
||||
@@ -478,6 +486,12 @@ func GetAllLogs(logType int, startTimestamp int64, endTimestamp int64, modelName
|
||||
if tokenName != "" {
|
||||
tx = tx.Where("logs.token_name = ?", tokenName)
|
||||
}
|
||||
if len(correlationKey) > 1 {
|
||||
tx = applyAttemptLogFilter(tx, correlationKey[1])
|
||||
}
|
||||
if len(correlationKey) > 0 && correlationKey[0] != "" {
|
||||
tx = tx.Where("logs.correlation_key = ?", correlationKey[0])
|
||||
}
|
||||
if requestId != "" {
|
||||
tx = tx.Where("logs.request_id = ?", requestId)
|
||||
}
|
||||
@@ -557,7 +571,7 @@ func GetAllLogs(logType int, startTimestamp int64, endTimestamp int64, modelName
|
||||
|
||||
const logSearchCountLimit = 10000
|
||||
|
||||
func GetUserLogs(userId int, logType int, startTimestamp int64, endTimestamp int64, modelName string, tokenName string, startIdx int, num int, group string, requestId string, upstreamRequestId string) (logs []*Log, total int64, err error) {
|
||||
func GetUserLogs(userId int, logType int, startTimestamp int64, endTimestamp int64, modelName string, tokenName string, startIdx int, num int, group string, requestId string, upstreamRequestId string, attemptStatus ...string) (logs []*Log, total int64, err error) {
|
||||
var tx *gorm.DB
|
||||
if logType == LogTypeUnknown {
|
||||
tx = LOG_DB.Where("logs.user_id = ?", userId)
|
||||
@@ -565,6 +579,10 @@ func GetUserLogs(userId int, logType int, startTimestamp int64, endTimestamp int
|
||||
tx = LOG_DB.Where("logs.user_id = ? and logs.type = ?", userId, logType)
|
||||
}
|
||||
|
||||
if len(attemptStatus) > 0 {
|
||||
tx = applyAttemptLogFilter(tx, attemptStatus[0])
|
||||
}
|
||||
|
||||
if tx, err = applyExplicitLogTextFilter(tx, "logs.model_name", modelName); err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
@@ -611,11 +629,15 @@ type Stat struct {
|
||||
Tpm int `json:"tpm"`
|
||||
}
|
||||
|
||||
func SumUsedQuota(logType int, startTimestamp int64, endTimestamp int64, modelName string, username string, tokenName string, channel int, group string) (stat Stat, err error) {
|
||||
func SumUsedQuota(logType int, startTimestamp int64, endTimestamp int64, modelName string, username string, tokenName string, channel int, group string, correlationKey ...string) (stat Stat, err error) {
|
||||
tx := LOG_DB.Table("logs").Select("COALESCE(sum(quota), 0) quota")
|
||||
|
||||
// 为rpm和tpm创建单独的查询
|
||||
rpmTpmQuery := LOG_DB.Table("logs").Select("count(*) rpm, COALESCE(sum(prompt_tokens), 0) + COALESCE(sum(completion_tokens), 0) tpm")
|
||||
if len(correlationKey) > 0 && correlationKey[0] != "" {
|
||||
tx = tx.Where("correlation_key = ?", correlationKey[0])
|
||||
rpmTpmQuery = rpmTpmQuery.Where("correlation_key = ?", correlationKey[0])
|
||||
}
|
||||
|
||||
if tx, err = applyExplicitLogTextFilter(tx, "username", username); err != nil {
|
||||
return stat, err
|
||||
@@ -651,6 +673,11 @@ func SumUsedQuota(logType int, startTimestamp int64, endTimestamp int64, modelNa
|
||||
tx = tx.Where("type = ?", LogTypeConsume)
|
||||
rpmTpmQuery = rpmTpmQuery.Where("type = ?", LogTypeConsume)
|
||||
|
||||
if len(correlationKey) > 1 {
|
||||
tx = applyAttemptLogFilter(tx, correlationKey[1])
|
||||
rpmTpmQuery = applyAttemptLogFilter(rpmTpmQuery, correlationKey[1])
|
||||
}
|
||||
|
||||
// 只统计最近60秒的rpm和tpm
|
||||
rpmTpmQuery = rpmTpmQuery.Where("created_at >= ?", time.Now().Add(-60*time.Second).Unix())
|
||||
|
||||
@@ -737,3 +764,15 @@ func DeleteOldLogBatch(ctx context.Context, targetTimestamp int64, limit int) (i
|
||||
}
|
||||
return result.RowsAffected, nil
|
||||
}
|
||||
|
||||
// A fixed top-level marker keeps filtering portable across SQL log stores.
|
||||
func applyAttemptLogFilter(tx *gorm.DB, status string) *gorm.DB {
|
||||
switch status {
|
||||
case "verified", "unchecked", "rejected", "charged_rejection", "switched_success", "mismatch_allowed", "transport_error":
|
||||
return tx.Where("other LIKE ?", `%"routing_status":"`+status+`"%`)
|
||||
case "estimated":
|
||||
return tx.Where("other LIKE ?", `%"attempt_usage_source":"estimated"%`)
|
||||
default:
|
||||
return tx
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user