feat: sync deployed v1.0.0-rc.38-custom.42
Sync production source including routing, billing, Astra detection and public uptime. Update custom build instructions and license packaging.
This commit is contained in:
cneicy committed
1 parent
aa6c3f50f4
commit
ceddfd3d6b
828 files changed
+122253
-13702
No files matched your search
@@ -0,0 +1,196 @@
|
||||
package controller
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/QuantumNous/new-api/common"
|
||||
"github.com/QuantumNous/new-api/constant"
|
||||
"github.com/QuantumNous/new-api/model"
|
||||
"github.com/QuantumNous/new-api/service"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestVLLMStatusPartialFailureAndCredentials(t *testing.T) {
|
||||
service.InitHttpClient()
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, http.MethodGet, r.Method)
|
||||
assert.Equal(t, "Bearer test-vllm-key", r.Header.Get("Authorization"))
|
||||
assert.Equal(t, "status-panel", r.Header.Get("X-Source"))
|
||||
switch r.URL.Path {
|
||||
case "/prefix/health":
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "/prefix/version":
|
||||
_, _ = w.Write([]byte(`{"version":"0.25.2"}`))
|
||||
case "/prefix/v1/models":
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"served-model","root":"source/model","max_model_len":1048576}]}`))
|
||||
case "/prefix/metrics":
|
||||
http.Error(w, "test-vllm-key should not reach the browser", http.StatusForbidden)
|
||||
default:
|
||||
t.Errorf("unexpected path: %s", r.URL.Path)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
base := upstream.URL + "/prefix/"
|
||||
override := `{"X-Source":"status-panel"}`
|
||||
channel := &model.Channel{Type: constant.ChannelTypeVLLM, BaseURL: &base, Key: "test-vllm-key", HeaderOverride: &override}
|
||||
status, err := fetchInferenceStatus(context.Background(), channel)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 200, status.Endpoints["/health"].Status)
|
||||
assert.Equal(t, "0.25.2", status.Version)
|
||||
require.Len(t, status.Models, 1)
|
||||
assert.Equal(t, "source/model", status.Models[0].Root)
|
||||
require.NotNil(t, status.Models[0].MaxModelLen)
|
||||
assert.EqualValues(t, 1048576, *status.Models[0].MaxModelLen)
|
||||
assert.Equal(t, inferenceEndpointStatus{Status: 403, Error: "http_error"}, status.Endpoints["/metrics"])
|
||||
assert.Empty(t, status.RawMetrics)
|
||||
assert.Empty(t, status.Metrics)
|
||||
body, err := common.Marshal(status)
|
||||
require.NoError(t, err)
|
||||
assert.NotContains(t, string(body), channel.Key)
|
||||
}
|
||||
|
||||
func TestVLLMStatusRejectsInvalidTargetsAndRedirects(t *testing.T) {
|
||||
service.InitHttpClient()
|
||||
for _, base := range []string{"", "file:///etc/passwd", "http://user:secret@localhost", "http://localhost?key=secret"} {
|
||||
_, err := fetchInferenceStatus(context.Background(), &model.Channel{Type: constant.ChannelTypeVLLM, BaseURL: &base})
|
||||
require.Error(t, err)
|
||||
assert.NotContains(t, err.Error(), "secret")
|
||||
}
|
||||
_, err := fetchInferenceStatus(context.Background(), &model.Channel{Type: constant.ChannelTypeOllama})
|
||||
require.Error(t, err)
|
||||
target := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
t.Error("redirect must not forward channel credentials")
|
||||
}))
|
||||
defer target.Close()
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Empty(t, r.Header.Get("Authorization"))
|
||||
http.Redirect(w, r, target.URL, http.StatusFound)
|
||||
}))
|
||||
defer upstream.Close()
|
||||
status, err := fetchInferenceStatus(context.Background(), &model.Channel{Type: constant.ChannelTypeVLLM, BaseURL: &upstream.URL, Key: "EMPTY"})
|
||||
require.NoError(t, err)
|
||||
for _, endpoint := range status.Endpoints {
|
||||
assert.Equal(t, inferenceEndpointStatus{Status: 302, Error: "http_error"}, endpoint)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVLLMStatusInvalidAndOversizedResponses(t *testing.T) {
|
||||
service.InitHttpClient()
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/health":
|
||||
w.WriteHeader(http.StatusServiceUnavailable)
|
||||
case "/version":
|
||||
_, _ = w.Write([]byte(`{"version":false}`))
|
||||
case "/v1/models":
|
||||
_, _ = w.Write([]byte(`{"data":[]}`))
|
||||
case "/metrics":
|
||||
_, _ = w.Write([]byte(strings.Repeat(" ", (4<<20)+1)))
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
channel := &model.Channel{Type: constant.ChannelTypeVLLM, BaseURL: &upstream.URL, Key: "EMPTY"}
|
||||
status, err := fetchInferenceStatus(context.Background(), channel)
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, 503, status.Endpoints["/health"].Status)
|
||||
assert.Equal(t, "invalid_response", status.Endpoints["/version"].Error)
|
||||
assert.Empty(t, status.Endpoints["/v1/models"].Error)
|
||||
assert.Equal(t, "response_too_large", status.Endpoints["/metrics"].Error)
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel()
|
||||
_, err = fetchInferenceStatus(ctx, channel)
|
||||
assert.ErrorIs(t, err, context.Canceled)
|
||||
}
|
||||
|
||||
func TestParseVLLMMetricsPreservesSeriesAndHistogramTotals(t *testing.T) {
|
||||
metrics, err := parseInferenceMetrics(`# TYPE vllm:num_requests_running gauge
|
||||
vllm:num_requests_running{engine="0",model_name="model,a"} 0
|
||||
vllm:num_requests_running{model_name="model,a",engine="1"} 2
|
||||
# TYPE vllm:time_to_first_token_seconds histogram
|
||||
vllm:time_to_first_token_seconds_bucket{le="+Inf"} 4
|
||||
vllm:time_to_first_token_seconds_sum 10
|
||||
vllm:time_to_first_token_seconds_count 4
|
||||
# TYPE vllm:kv_cache_usage_perc gauge
|
||||
vllm:kv_cache_usage_perc NaN
|
||||
vllm:bad_value +Inf
|
||||
# TYPE process_start_time_seconds gauge
|
||||
process_start_time_seconds 100
|
||||
unrelated_metric 123
|
||||
`, "vllm:")
|
||||
require.NoError(t, err)
|
||||
assert.ElementsMatch(t, []inferenceMetric{
|
||||
{Name: "vllm:num_requests_running", Labels: map[string]string{"engine": "0", "model_name": "model,a"}, Value: 0},
|
||||
{Name: "vllm:num_requests_running", Labels: map[string]string{"engine": "1", "model_name": "model,a"}, Value: 2},
|
||||
{Name: "vllm:time_to_first_token_seconds_sum", Labels: map[string]string{}, Value: 10},
|
||||
{Name: "vllm:time_to_first_token_seconds_count", Labels: map[string]string{}, Value: 4},
|
||||
{Name: "process_start_time_seconds", Labels: map[string]string{}, Value: 100},
|
||||
}, metrics)
|
||||
_, err = parseInferenceMetrics("vllm:broken{engine=oops} 1\n", "vllm:")
|
||||
require.Error(t, err)
|
||||
}
|
||||
|
||||
// Opt-in verification against an operator-provided instance; credentials stay in the environment.
|
||||
func TestVLLMStatusLive(t *testing.T) {
|
||||
base := os.Getenv("NEW_API_VLLM_TEST_URL")
|
||||
if base == "" {
|
||||
t.Skip("NEW_API_VLLM_TEST_URL is not configured")
|
||||
}
|
||||
service.InitHttpClient()
|
||||
status, err := fetchInferenceStatus(context.Background(), &model.Channel{
|
||||
Type: constant.ChannelTypeVLLM, BaseURL: &base, Key: os.Getenv("NEW_API_VLLM_TEST_KEY"),
|
||||
})
|
||||
require.NoError(t, err)
|
||||
for path, endpoint := range status.Endpoints {
|
||||
assert.Equal(t, 200, endpoint.Status, path)
|
||||
assert.Empty(t, endpoint.Error, path)
|
||||
}
|
||||
assert.NotEmpty(t, status.Version)
|
||||
assert.NotEmpty(t, status.Models)
|
||||
assert.NotEmpty(t, status.Metrics)
|
||||
assert.NotEmpty(t, status.RawMetrics)
|
||||
}
|
||||
|
||||
func TestSGLangStatusUsesDocumentedEndpointsAndRedactsServerConfiguration(t *testing.T) {
|
||||
service.InitHttpClient()
|
||||
upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
assert.Equal(t, http.MethodGet, r.Method)
|
||||
assert.Equal(t, "Bearer test-key", r.Header.Get("Authorization"))
|
||||
switch r.URL.Path {
|
||||
case "/health":
|
||||
w.WriteHeader(http.StatusOK)
|
||||
case "/server_info":
|
||||
_, _ = w.Write([]byte(`{"version":"0.5.19","api_key":"server-secret","admin_api_key":"admin-secret","internal_states":[{"private":"private-state"}]}`))
|
||||
case "/v1/models":
|
||||
_, _ = w.Write([]byte(`{"data":[{"id":"served-model","root":"served-model","max_model_len":32768}]}`))
|
||||
case "/metrics":
|
||||
_, _ = w.Write([]byte("# TYPE sglang:num_running_reqs gauge\nsglang:num_running_reqs{dp_rank=\"0\"} 0\nsglang:num_running_reqs{dp_rank=\"1\"} 2\n# TYPE sglang:time_to_first_token_seconds histogram\nsglang:time_to_first_token_seconds_bucket{le=\"+Inf\"} 2\nsglang:time_to_first_token_seconds_sum 1.5\nsglang:time_to_first_token_seconds_count 2\nvllm:num_requests_running 9\n"))
|
||||
default:
|
||||
t.Errorf("unexpected SGLang path: %s", r.URL.Path)
|
||||
w.WriteHeader(http.StatusNotFound)
|
||||
}
|
||||
}))
|
||||
defer upstream.Close()
|
||||
status, err := fetchInferenceStatus(context.Background(), &model.Channel{Type: constant.ChannelTypeSGLang, BaseURL: &upstream.URL, Key: "test-key"})
|
||||
require.NoError(t, err)
|
||||
assert.Equal(t, "0.5.19", status.Version)
|
||||
require.Len(t, status.Models, 1)
|
||||
assert.Equal(t, "served-model", status.Models[0].ID)
|
||||
assert.ElementsMatch(t, []inferenceMetric{
|
||||
{Name: "sglang:num_running_reqs", Labels: map[string]string{"dp_rank": "0"}, Value: 0},
|
||||
{Name: "sglang:num_running_reqs", Labels: map[string]string{"dp_rank": "1"}, Value: 2},
|
||||
{Name: "sglang:time_to_first_token_seconds_sum", Labels: map[string]string{}, Value: 1.5},
|
||||
{Name: "sglang:time_to_first_token_seconds_count", Labels: map[string]string{}, Value: 2},
|
||||
}, status.Metrics)
|
||||
encoded, err := common.Marshal(status)
|
||||
require.NoError(t, err)
|
||||
for _, secret := range []string{"test-key", "server-secret", "admin-secret", "private-state", "api_key", "internal_states"} {
|
||||
assert.NotContains(t, string(encoded), secret)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user