/* Copyright (C) 2023-2026 QuantumNous This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details. You should have received a copy of the GNU Affero General Public License along with this program. If not, see . For commercial licensing, please contact support@quantumnous.com */ import { useQuery } from '@tanstack/react-query' import { useId, useRef, useState } from 'react' import { useTranslation } from 'react-i18next' import { Dialog } from '@/components/dialog' import { ErrorState } from '@/components/error-state' import { LoadingState } from '@/components/loading-state' import { Alert, AlertDescription } from '@/components/ui/alert' import { Button } from '@/components/ui/button' import { Label } from '@/components/ui/label' import { Switch } from '@/components/ui/switch' import { toIntlLocale } from '@/i18n/languages' import { ADMIN_PERMISSION_ACTIONS, ADMIN_PERMISSION_RESOURCES, hasPermission, } from '@/lib/admin-permissions' import { getServerErrorMessage } from '@/lib/server-error-message' import { useAuthStore } from '@/stores/auth-store' import { getInferenceStatus } from '../../api' import { metricMean, metricValue, metricRatio, recentMetrics, type InferenceStatus, } from '../../lib/inference-status' type MetricRow = { label: string details?: string value: number | undefined unit?: 'percent' | 'seconds' | 'rate' } type InferenceStatusDialogProps = { provider: 'vllm' | 'sglang' channelId: number channelName: string onClose: () => void onSyncModels: () => void onTestChannel: () => void } // Mounted only while this channel's status dialog is open; unmount aborts the query. export function InferenceStatusDialog(props: InferenceStatusDialogProps) { const { t, i18n } = useTranslation() const isSGLang = props.provider === 'sglang' const [autoRefresh, setAutoRefresh] = useState(true) const autoRefreshId = useId() const previous = useRef< | { channelId: number; provider: string; snapshot: InferenceStatus } | undefined >(undefined) const user = useAuthStore((state) => state.auth.user) const canOperate = hasPermission( user, ADMIN_PERMISSION_RESOURCES.CHANNEL, ADMIN_PERMISSION_ACTIONS.OPERATE ) const canSync = canOperate && hasPermission( user, ADMIN_PERMISSION_RESOURCES.CHANNEL, ADMIN_PERMISSION_ACTIONS.WRITE ) const query = useQuery({ queryKey: ['channels', props.channelId, props.provider, 'inference-status'], queryFn: async ({ signal }) => { const snapshot = await getInferenceStatus( props.channelId, props.provider, signal ) signal.throwIfAborted() const last = previous.current?.channelId === props.channelId && previous.current.provider === props.provider ? previous.current.snapshot : undefined const recent = recentMetrics(snapshot, last) previous.current = { channelId: props.channelId, provider: props.provider, snapshot, } return { snapshot, recent } }, refetchInterval: autoRefresh ? 5000 : false, refetchIntervalInBackground: false, refetchOnWindowFocus: false, retry: false, gcTime: 0, meta: { errorToast: false }, }) const snapshot = query.data?.snapshot const metrics = snapshot?.metrics ?? [] const recent = query.data?.recent const locale = toIntlLocale(i18n.resolvedLanguage || i18n.language) const number = new Intl.NumberFormat(locale, { maximumFractionDigits: 2, }) const percent = new Intl.NumberFormat(locale, { style: 'percent', maximumFractionDigits: 2, }) const seconds = new Intl.NumberFormat(locale, { style: 'unit', unit: 'second', maximumFractionDigits: 2, }) const groups: { title: string; hint?: string; rows: MetricRow[] }[] = [ { title: t('Current load'), rows: [ { label: t('Running requests'), value: metricValue( metrics, isSGLang ? 'sglang:num_running_reqs' : 'vllm:num_requests_running' ), }, { label: t('Waiting requests'), value: metricValue( metrics, isSGLang ? 'sglang:num_queue_reqs' : 'vllm:num_requests_waiting' ), }, { label: isSGLang ? t('Peak token pool usage') : t('Peak KV cache usage'), value: isSGLang ? metricValue(metrics, 'sglang:token_usage', {}, 'max') : (metricValue(metrics, 'vllm:kv_cache_usage_perc', {}, 'max') ?? metricValue(metrics, 'vllm:gpu_cache_usage_perc', {}, 'max')), unit: 'percent', }, ...(isSGLang ? [ { label: t('Output token rate'), value: metricValue(metrics, 'sglang:gen_throughput'), unit: 'rate' as const, }, ] : [ { label: t('Awake engines'), value: metricValue(metrics, 'vllm:engine_sleep_state', { sleep_state: 'awake', }), }, ]), ], }, { title: t('Cumulative usage'), hint: t( 'Counters and averages cover the period since the upstream metrics were initialized or reset.' ), rows: [ { label: isSGLang ? t('Processed requests') : t('Finished requests'), value: metricValue( metrics, isSGLang ? 'sglang:num_requests_total' : 'vllm:request_success_total' ), }, { label: t('Input tokens'), value: metricValue(metrics, `${props.provider}:prompt_tokens_total`), }, { label: t('Output tokens'), value: metricValue( metrics, `${props.provider}:generation_tokens_total` ), }, ...(isSGLang ? [ { label: t('Cached tokens'), value: metricValue(metrics, 'sglang:cached_tokens_total'), }, ] : [ { label: t('Prefix cache hit rate'), value: metricRatio( metricValue(metrics, 'vllm:prefix_cache_hits_total'), metricValue(metrics, 'vllm:prefix_cache_queries_total') ), unit: 'percent' as const, }, { label: t('Speculative decoding acceptance rate'), value: metricRatio( metricValue( metrics, 'vllm:spec_decode_num_accepted_tokens_total' ), metricValue( metrics, 'vllm:spec_decode_num_draft_tokens_total' ) ), unit: 'percent' as const, }, ]), { label: t('Mean time to first token'), value: metricMean( metrics, `${props.provider}:time_to_first_token_seconds` ), unit: 'seconds', }, { label: t('Mean request latency'), value: metricMean( metrics, `${props.provider}:e2e_request_latency_seconds` ), unit: 'seconds', }, { label: t('Mean queue time'), value: metricMean( metrics, isSGLang ? 'sglang:queue_time_seconds' : 'vllm:request_queue_time_seconds' ), unit: 'seconds', }, { label: t('Mean time per output token'), value: isSGLang ? (metricMean(metrics, 'sglang:inter_token_latency_seconds') ?? metricMean(metrics, 'sglang:time_per_output_token_seconds')) : metricMean(metrics, 'vllm:request_time_per_output_token_seconds'), unit: 'seconds', }, ], }, { title: t('Since previous sample'), hint: recent ? t('Sample interval: {{seconds}} seconds', { seconds: number.format(recent.seconds), }) : t( 'Waiting for two comparable samples. Counter resets restart the sampling window.' ), rows: [ { label: t('Input token rate'), value: metricRatio( metricValue( recent?.metrics ?? [], `${props.provider}:prompt_tokens_total` ), recent?.seconds ), unit: 'rate', }, { label: t('Output token rate'), value: metricRatio( metricValue( recent?.metrics ?? [], `${props.provider}:generation_tokens_total` ), recent?.seconds ), unit: 'rate', }, { label: t('Mean time to first token'), value: metricMean( recent?.metrics ?? [], `${props.provider}:time_to_first_token_seconds` ), unit: 'seconds', }, { label: t('Mean request latency'), value: metricMean( recent?.metrics ?? [], `${props.provider}:e2e_request_latency_seconds` ), unit: 'seconds', }, ], }, ] if (isSGLang) { // These SGLang values are per-worker gauges, not cumulative counters. // Keep each labelled series rather than adding or averaging percentages. const workerMetrics = metrics.filter((metric) => ['sglang:cache_hit_rate', 'sglang:spec_accept_rate'].includes(metric.name) ) if (workerMetrics.length) { groups.push({ title: t('Worker metrics'), rows: workerMetrics.map((metric) => ({ label: metric.name === 'sglang:cache_hit_rate' ? t('Prefix cache hit rate') : t('Speculative decoding acceptance rate'), details: JSON.stringify(metric.labels), value: metric.value, unit: 'percent', })), }) } } const handleExport = () => { if (!snapshot) return const blob = new Blob([JSON.stringify(snapshot, null, 2)], { type: 'application/json', }) const url = URL.createObjectURL(blob) const link = document.createElement('a') link.href = url link.download = `${props.provider}-${props.channelId}-${snapshot.sampled_at}.json` link.click() URL.revokeObjectURL(url) } return ( { if (!open) props.onClose() }} title={isSGLang ? t('SGLang status') : t('vLLM status')} description={props.channelName} contentClassName='sm:max-w-3xl' bodyClassName='space-y-5' footer={ } >
{isSGLang && snapshot?.endpoints['/metrics']?.error && ( {t( 'Start SGLang with --enable-metrics to expose Prometheus metrics.' )} )} {query.isPending && } {query.isError && !snapshot && ( void query.refetch()} /> )} {query.isError && snapshot && ( {t('Refresh failed. Showing the last successful snapshot.')} )} {snapshot && ( <>
{t('Version')}:{' '} {snapshot.version || t('Unavailable')} {t('Last updated')}:{' '} {new Date(snapshot.sampled_at).toLocaleString(locale)}
{Object.entries(snapshot.endpoints).map(([path, endpoint]) => (
{path}
{endpoint.error ? t('Unavailable') : t('Available')} {endpoint.status > 0 && ` · HTTP ${endpoint.status}`} {endpoint.error === 'invalid_response' && ` · ${t('Invalid response')}`} {endpoint.error === 'response_too_large' && ` · ${t('Response too large')}`}
))}

{t('Models')}

{snapshot.models.length === 0 && (

{snapshot.endpoints['/v1/models']?.error ? t('Unavailable') : t('No models found')}

)} {snapshot.models.map((model) => (

{model.id}

{t('Model source')}: {model.root || t('Unavailable')}

{t('Max context tokens')}:{' '} {model.max_model_len == null ? t('Unavailable') : number.format(model.max_model_len)}

))}
{groups.map((group) => (

{group.title}

{group.hint && (

{group.hint}

)}
{group.rows.map((row) => { let value = t('Unavailable') if (row.value !== undefined) { value = number.format(row.value) if (row.unit === 'percent') { value = percent.format(row.value) } if (row.unit === 'seconds') { value = seconds.format(row.value) } if (row.unit === 'rate') { value = t('{{value}} tokens/s', { value: number.format(row.value), }) } } return (
{row.label}
{value}
{row.details && (
{row.details}
)}
) })}
))} )}
) }