feat: sync deployed v1.0.0-rc.38-custom.42
Sync production source including routing, billing, Astra detection and public uptime. Update custom build instructions and license packaging.
This commit is contained in:
cneicy committed
1 parent
aa6c3f50f4
commit
ceddfd3d6b
828 files changed
+122253
-13702
No files matched your search
@@ -0,0 +1,242 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"io"
|
||||
"math"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
)
|
||||
|
||||
type inferenceEndpointStatus struct {
|
||||
Status int `json:"status"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type inferenceModel struct {
|
||||
ID string `json:"id"`
|
||||
Root string `json:"root"`
|
||||
MaxModelLen *int64 `json:"max_model_len"`
|
||||
}
|
||||
|
||||
type inferenceMetric struct {
|
||||
Name string `json:"name"`
|
||||
Labels map[string]string `json:"labels"`
|
||||
Value float64 `json:"value"`
|
||||
}
|
||||
|
||||
type inferenceStatus struct {
|
||||
SampledAt int64 `json:"sampled_at"`
|
||||
Endpoints map[string]inferenceEndpointStatus `json:"endpoints"`
|
||||
Version string `json:"version"`
|
||||
Models []inferenceModel `json:"models"`
|
||||
Metrics []inferenceMetric `json:"metrics"`
|
||||
RawMetrics string `json:"raw_metrics"`
|
||||
}
|
||||
|
||||
func GetVLLMChannelStatus(c *gin.Context) {
|
||||
getInferenceChannelStatus(c, constant.ChannelTypeVLLM)
|
||||
}
|
||||
|
||||
func GetSGLangChannelStatus(c *gin.Context) {
|
||||
getInferenceChannelStatus(c, constant.ChannelTypeSGLang)
|
||||
}
|
||||
|
||||
func getInferenceChannelStatus(c *gin.Context, channelType int) {
|
||||
c.Header("Cache-Control", "no-store")
|
||||
id, err := strconv.Atoi(c.Param("id"))
|
||||
if err != nil || id <= 0 {
|
||||
c.JSON(http.StatusBadRequest, gin.H{"success": false, "message": "Invalid channel id"})
|
||||
return
|
||||
}
|
||||
channel, err := model.GetChannelById(id, true)
|
||||
if err != nil || channel == nil {
|
||||
c.JSON(http.StatusNotFound, gin.H{"success": false, "message": "Channel not found"})
|
||||
return
|
||||
}
|
||||
if channel.Type != channelType {
|
||||
common.ApiError(c, errors.New("Channel type does not match the status endpoint"))
|
||||
return
|
||||
}
|
||||
status, err := fetchInferenceStatus(c.Request.Context(), channel)
|
||||
if err != nil {
|
||||
common.ApiError(c, err)
|
||||
return
|
||||
}
|
||||
common.ApiSuccess(c, status)
|
||||
}
|
||||
|
||||
// fetchInferenceStatus reads only fixed paths on an operator-configured channel.
|
||||
// Each endpoint can fail independently; upstream errors never echo credentials.
|
||||
func fetchInferenceStatus(ctx context.Context, channel *model.Channel) (*inferenceStatus, error) {
|
||||
if channel.Type != constant.ChannelTypeVLLM && channel.Type != constant.ChannelTypeSGLang {
|
||||
return nil, errors.New("This operation is only supported for vLLM or SGLang channels")
|
||||
}
|
||||
baseURL := strings.TrimRight(strings.TrimSpace(channel.GetBaseURL()), "/")
|
||||
parsedURL, err := url.Parse(baseURL)
|
||||
if err != nil || parsedURL.Host == "" || parsedURL.User != nil || parsedURL.RawQuery != "" || parsedURL.Fragment != "" || (parsedURL.Scheme != "http" && parsedURL.Scheme != "https") {
|
||||
return nil, errors.New("Invalid inference server address")
|
||||
}
|
||||
key, _, keyErr := channel.GetNextEnabledKey()
|
||||
if keyErr != nil {
|
||||
return nil, errors.New("No enabled channel key")
|
||||
}
|
||||
key = strings.TrimSpace(key)
|
||||
headers, err := buildFetchModelsHeaders(channel, key)
|
||||
if err != nil {
|
||||
return nil, errors.New("Invalid channel header override")
|
||||
}
|
||||
if (key == "" || key == "EMPTY") && headers.Get("Authorization") == "Bearer "+key {
|
||||
headers.Del("Authorization")
|
||||
}
|
||||
settings := channel.GetSetting()
|
||||
client, err := service.GetHttpClientWithProxySettings(settings.Proxy, settings)
|
||||
if err != nil {
|
||||
return nil, errors.New("Invalid channel proxy")
|
||||
}
|
||||
// Do not mutate the shared client or forward channel credentials on redirects.
|
||||
statusClient := *client
|
||||
statusClient.CheckRedirect = func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }
|
||||
ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
|
||||
defer cancel()
|
||||
|
||||
versionPath, metricPrefix := "/version", "vllm:"
|
||||
if channel.Type == constant.ChannelTypeSGLang {
|
||||
versionPath, metricPrefix = "/server_info", "sglang:"
|
||||
}
|
||||
paths := []string{"/health", versionPath, "/v1/models", "/metrics"}
|
||||
results := make([]struct {
|
||||
inferenceEndpointStatus
|
||||
body []byte
|
||||
at int64
|
||||
}, len(paths))
|
||||
var wg sync.WaitGroup
|
||||
for i, path := range paths {
|
||||
wg.Go(func() {
|
||||
result := &results[i]
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, baseURL+path, nil)
|
||||
if err != nil {
|
||||
result.Error = "request_failed"
|
||||
return
|
||||
}
|
||||
req.Header = headers.Clone()
|
||||
req.Host = headers.Get("Host")
|
||||
resp, err := statusClient.Do(req)
|
||||
if err != nil {
|
||||
result.Error = "request_failed"
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
result.Status = resp.StatusCode
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
result.Error = "http_error"
|
||||
return
|
||||
}
|
||||
const maxResponseBytes = 4 << 20
|
||||
body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1))
|
||||
if err != nil {
|
||||
result.Error = "request_failed"
|
||||
return
|
||||
}
|
||||
if len(body) > maxResponseBytes {
|
||||
result.Error = "response_too_large"
|
||||
return
|
||||
}
|
||||
result.body = body
|
||||
result.at = time.Now().UnixMilli()
|
||||
})
|
||||
}
|
||||
wg.Wait()
|
||||
if errors.Is(ctx.Err(), context.Canceled) {
|
||||
return nil, context.Canceled
|
||||
}
|
||||
status := &inferenceStatus{
|
||||
SampledAt: time.Now().UnixMilli(),
|
||||
Endpoints: make(map[string]inferenceEndpointStatus, len(paths)),
|
||||
Models: []inferenceModel{},
|
||||
Metrics: []inferenceMetric{},
|
||||
}
|
||||
for i, path := range paths {
|
||||
result := &results[i]
|
||||
if result.Error == "" {
|
||||
switch path {
|
||||
case versionPath:
|
||||
// /server_info includes the launch configuration. Decode only the
|
||||
// version so API keys and other server settings never reach the browser.
|
||||
var version struct {
|
||||
Version string `json:"version"`
|
||||
}
|
||||
if err := common.Unmarshal(result.body, &version); err != nil || version.Version == "" {
|
||||
result.Error = "invalid_response"
|
||||
} else {
|
||||
status.Version = version.Version
|
||||
}
|
||||
case "/v1/models":
|
||||
var models struct {
|
||||
Data []inferenceModel `json:"data"`
|
||||
}
|
||||
if err := common.Unmarshal(result.body, &models); err != nil || models.Data == nil {
|
||||
result.Error = "invalid_response"
|
||||
} else {
|
||||
status.Models = models.Data
|
||||
}
|
||||
case "/metrics":
|
||||
metrics, err := parseInferenceMetrics(string(result.body), metricPrefix)
|
||||
if err != nil {
|
||||
result.Error = "invalid_response"
|
||||
} else {
|
||||
status.Metrics = metrics
|
||||
status.RawMetrics = string(result.body)
|
||||
status.SampledAt = result.at
|
||||
}
|
||||
}
|
||||
}
|
||||
status.Endpoints[path] = result.inferenceEndpointStatus
|
||||
}
|
||||
return status, nil
|
||||
}
|
||||
|
||||
func parseInferenceMetrics(raw string, prefix string) ([]inferenceMetric, error) {
|
||||
var parser expfmt.TextParser
|
||||
families, err := parser.TextToMetricFamilies(strings.NewReader(raw))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
metrics := []inferenceMetric{}
|
||||
for name, family := range families {
|
||||
if !strings.HasPrefix(name, prefix) && name != "process_start_time_seconds" {
|
||||
continue
|
||||
}
|
||||
samples, err := expfmt.ExtractSamples(&expfmt.DecodeOptions{}, family)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, sample := range samples {
|
||||
name := string(sample.Metric["__name__"])
|
||||
value := float64(sample.Value)
|
||||
if strings.HasSuffix(name, "_bucket") || math.IsNaN(value) || math.IsInf(value, 0) || value < 0 {
|
||||
continue
|
||||
}
|
||||
labels := make(map[string]string, len(sample.Metric)-1)
|
||||
for label, value := range sample.Metric {
|
||||
if label != "__name__" {
|
||||
labels[string(label)] = string(value)
|
||||
}
|
||||
}
|
||||
metrics = append(metrics, inferenceMetric{Name: name, Labels: labels, Value: value})
|
||||
}
|
||||
}
|
||||
return metrics, nil
|
||||
}
|
||||
Reference in new issue
Block a user