feat(dashboard): add vLLM token throughput monitoring (#14325)

frontend/src/views/home/index.vue

@@ -99,6 +99,7 @@
                         >
                             <el-radio-button value="network">{{ $t('home.network') }}</el-radio-button>
                             <el-radio-button value="io">{{ $t('home.io') }}</el-radio-button>
+                            <el-radio-button v-if="hasVllm" value="vllm">vLLM</el-radio-button>
                         </el-radio-group>
                         <el-select
                             v-if="chartOption === 'network'"
@@ -128,6 +129,26 @@
                                 :value="item"
                             />
                         </el-select>
+                        <el-select
+                            v-if="chartOption === 'vllm'"
+                            v-model="vllmInstallID"
+                            class="p-w-200 float-right"
+                            :placeholder="$t('aiTools.model.deployments')"
+                            @visible-change="(visible) => visible && loadVllmOptions()"
+                        >
+                            <el-option
+                                v-for="item in vllmOptions"
+                                :key="item.appInstallId"
+                                :label="item.name + (item.servedModelName ? ' · ' + item.servedModelName : '')"
+                                :value="item.appInstallId"
+                                :disabled="item.status !== 'Running'"
+                            >
+                                <span>
+                                    {{ item.name }}{{ item.servedModelName ? ' · ' + item.servedModelName : '' }}
+                                </span>
+                                <Status class="ml-2" :status="item.status" />
+                            </el-option>
+                        </el-select>
                     </template>
                     <template #body>
                         <div class="monitor-chart-content">
@@ -169,6 +190,47 @@
                                     :dataZoom="true"
                                 />
                             </div>
+                            <template v-if="chartOption === 'vllm'">
+                                <div class="monitor-tags" :style="monitorTagsStyle">
+                                    <el-tag>
+                                        {{ $t('aiTools.vllmMonitor.promptThroughput') }}:
+                                        {{ formatVllmMetric(vllmCurrent?.promptThroughput) }} tokens/s
+                                    </el-tag>
+                                    <el-tag>
+                                        {{ $t('aiTools.vllmMonitor.generationThroughput') }}:
+                                        {{ formatVllmMetric(vllmCurrent?.generationThroughput) }} tokens/s
+                                    </el-tag>
+                                    <el-tag>
+                                        {{ $t('aiTools.vllmMonitor.processing') }}:
+                                        {{ formatVllmMetric(vllmCurrent?.running) }}
+                                    </el-tag>
+                                    <el-tag>
+                                        {{ $t('aiTools.vllmMonitor.queued') }}:
+                                        {{ formatVllmMetric(vllmCurrent?.waiting) }}
+                                    </el-tag>
+                                </div>
+                                <div class="mobile-monitor-chart">
+                                    <v-charts
+                                        height="383px"
+                                        id="vllmChart"
+                                        type="line"
+                                        :option="vllmChartOption"
+                                        :dataZoom="true"
+                                    />
+                                </div>
+                                <div class="vllm-monitor-footer">
+                                    <span>{{ vllmMonitorMessage }}</span>
+                                    <el-button
+                                        v-if="selectedVllm"
+                                        link
+                                        type="primary"
+                                        @click="vllmMonitorRef?.acceptParams(selectedVllm)"
+                                    >
+                                        {{ $t('commons.button.view') }}
+                                    </el-button>
+                                </div>
+                                <VllmMonitor v-if="selectedVllm" ref="vllmMonitorRef" />
+                            </template>
                         </div>
                     </template>
                 </CardWithHeader>
@@ -443,7 +505,7 @@
 </template>
 
 <script lang="ts" setup>
-import { onMounted, onBeforeUnmount, ref, reactive, computed, nextTick } from 'vue';
+import { onMounted, onBeforeUnmount, ref, reactive, computed, nextTick, watch, defineAsyncComponent } from 'vue';
 import SystemStatus from '@/views/home/status/index.vue';
 import AppLauncher from '@/views/home/app/index.vue';
 import VCharts from '@/components/v-charts/index.vue';
@@ -478,6 +540,9 @@ import {
 } from '@/utils/dashboardCache';
 import { MsgSuccess } from '@/utils/message';
 import { useCan } from '@/composables/useMenuManagePermission';
+import { readPages } from '@/utils/pagination';
+import { loadExtensionComponent } from '@/extensions/optional';
+import type { ReqPage, ResPage, ResultData } from '@/api/interface';
 const router = useRouter();
 import { useGlobalStore } from '@/composables/useGlobalStore';
 const {
@@ -489,8 +554,136 @@ const {
     hasNewVersion,
     isAdminOrNodeAdmin,
     isXpackOrEE,
+    isProductPro,
+    currentNode,
 } = useGlobalStore();
 
+interface VllmDeployment {
+    appInstallId: number;
+    name: string;
+    servedModelName: string;
+    status: string;
+}
+interface VllmSnapshot {
+    appInstallID: number;
+    createdAt: string;
+    status: string;
+    intervalSeconds: number;
+    promptThroughput: number | null;
+    generationThroughput: number | null;
+    running: number | null;
+    waiting: number | null;
+    metrics?: Record<string, Record<string, number>>;
+}
+const vllmModules = import.meta.glob<{
+    pageVllm: (req: ReqPage & { info: string }) => Promise<ResultData<ResPage<VllmDeployment>>>;
+    loadVllmCurrent: (id: number, signal: AbortSignal) => Promise<ResultData<VllmSnapshot>>;
+}>('@/xpack/api/modules/vllm.ts', { eager: true });
+const vllmMetricsModules = import.meta.glob<{
+    calculateCurrentMetrics: (snapshot: VllmSnapshot, previous?: VllmSnapshot) => VllmSnapshot;
+}>('@/xpack/views/vllm/monitor/current.ts', { eager: true });
+const vllmAPI = Object.values(vllmModules)[0];
+const vllmMetrics = Object.values(vllmMetricsModules)[0];
+const VllmMonitor = defineAsyncComponent(() => loadExtensionComponent('/vllm/monitor/index.vue'));
+const vllmMonitorRef = ref();
+const canViewVllm = useCan('ai_model_view');
+const vllmOptions = ref<VllmDeployment[]>([]);
+const vllmInstallID = ref<number>();
+const vllmCurrent = ref<VllmSnapshot>();
+const vllmLoadFailed = ref(false);
+const vllmPoints = ref<Array<{ time: string; prompt: number | null; generation: number | null }>>([]);
+let vllmTimer: ReturnType<typeof setTimeout>;
+let vllmController: AbortController;
+let vllmOptionsVersion = 0;
+const hasVllm = computed(() => isProductPro.value && canViewVllm.value && vllmOptions.value.length > 0);
+const selectedVllm = computed(() => vllmOptions.value.find((item) => item.appInstallId === vllmInstallID.value));
+const formatVllmMetric = (value?: number | null) =>
+    value == null || !Number.isFinite(value) ? '--' : Number(value.toFixed(2)).toString();
+const vllmMonitorMessage = computed(() => {
+    if (!selectedVllm.value) return i18n.global.t('aiTools.model.readinessNotRunning');
+    if (vllmLoadFailed.value) return i18n.global.t('aiTools.vllmMonitor.loadFailed');
+    if (vllmCurrent.value?.status === 'unavailable') return i18n.global.t('aiTools.vllmMonitor.unavailable');
+    if (vllmCurrent.value?.status === 'no_data') return i18n.global.t('aiTools.vllmMonitor.noMetrics');
+    if (!vllmCurrent.value?.intervalSeconds) return i18n.global.t('commons.table.statusWaiting');
+    return `${i18n.global.t('aiTools.vllmMonitor.sampleWindow')}: ${formatVllmMetric(vllmCurrent.value.intervalSeconds)} s`;
+});
+const vllmChartOption = computed(() => ({
+    xData: loadMonitorChartTime(vllmPoints.value.map((point) => point.time)),
+    yData: [
+        {
+            name: i18n.global.t('aiTools.vllmMonitor.promptThroughput'),
+            data: loadMonitorChartData(vllmPoints.value.map((point) => point.prompt)),
+        },
+        {
+            name: i18n.global.t('aiTools.vllmMonitor.generationThroughput'),
+            data: loadMonitorChartData(vllmPoints.value.map((point) => point.generation)),
+        },
+    ],
+    grid: monitorChartGrid,
+    legend: monitorChartLegend,
+    formatStr: 'tokens/s',
+    tooltip: {
+        trigger: 'axis',
+        valueFormatter: (value: number | null) => `${formatVllmMetric(value)} tokens/s`,
+    },
+}));
+
+const loadVllmOptions = async () => {
+    if (!isProductPro.value || !canViewVllm.value || !vllmAPI || !vllmMetrics) return;
+    const version = ++vllmOptionsVersion;
+    const node = currentNode.value;
+    try {
+        const items = await readPages((page) => {
+            if (version !== vllmOptionsVersion || node !== currentNode.value) throw new Error('node changed');
+            return vllmAPI.pageVllm({ ...page, info: '' });
+        });
+        if (version !== vllmOptionsVersion || node !== currentNode.value) return;
+        vllmOptions.value = items;
+        if (!items.some((item) => item.appInstallId === vllmInstallID.value && item.status === 'Running')) {
+            vllmInstallID.value = items.find((item) => item.status === 'Running')?.appInstallId;
+        }
+        if (!items.length && chartOption.value === 'vllm') chartOption.value = 'network';
+    } catch {
+        if (version === vllmOptionsVersion) vllmLoadFailed.value = true;
+    }
+};
+
+const stopVllmMonitor = () => {
+    clearTimeout(vllmTimer);
+    vllmController?.abort();
+    vllmCurrent.value = undefined;
+    vllmPoints.value = [];
+    vllmLoadFailed.value = false;
+};
+
+const refreshVllmMonitor = async () => {
+    const controller = new AbortController();
+    vllmController = controller;
+    try {
+        const { data } = await vllmAPI.loadVllmCurrent(vllmInstallID.value, controller.signal);
+        if (controller.signal.aborted) return;
+        const wasUnavailable = vllmCurrent.value?.status === 'unavailable';
+        vllmCurrent.value = vllmMetrics.calculateCurrentMetrics(data, vllmCurrent.value);
+        vllmLoadFailed.value = false;
+        vllmPoints.value.push({
+            time: dateFormatForSecond(new Date(data.createdAt)),
+            prompt: vllmCurrent.value.promptThroughput,
+            generation: vllmCurrent.value.generationThroughput,
+        });
+        if (data.status === 'unavailable' && !wasUnavailable) void loadVllmOptions();
+    } catch {
+        if (controller.signal.aborted) return;
+        vllmCurrent.value = undefined;
+        vllmLoadFailed.value = true;
+        vllmPoints.value.push({ time: dateFormatForSecond(new Date()), prompt: null, generation: null });
+    } finally {
+        if (!controller.signal.aborted) {
+            vllmPoints.value = vllmPoints.value.slice(-20);
+            vllmTimer = setTimeout(refreshVllmMonitor, 3000);
+        }
+    }
+};
+
 const DASHBOARD_CACHE_TTL = {
     safeStatus: 10 * 60 * 1000,
     netOptions: 60 * 60 * 1000,
@@ -505,7 +698,7 @@ const monitorTagsStyle = {
 const monitorChartEmptyLength = 20;
 const loadMonitorEmptyData = () => Array.from({ length: monitorChartEmptyLength }, () => null);
 const loadMonitorEmptyTime = () => Array.from({ length: monitorChartEmptyLength }, () => '');
-const loadMonitorChartData = (data: Array<number>) => (data.length === 0 ? loadMonitorEmptyData() : data);
+const loadMonitorChartData = (data: Array<number | null>) => (data.length === 0 ? loadMonitorEmptyData() : data);
 const loadMonitorChartTime = (data: Array<string>) => (data.length === 0 ? loadMonitorEmptyTime() : data);
 const loadIOChartOption = () => ({
     xData: loadMonitorChartTime(timeIODatas.value),
@@ -718,6 +911,7 @@ const updateCurrentInfo = (data: Dashboard.CurrentInfo) => {
 
 const changeOption = async () => {
     isInit.value = true;
+    if (chartOption.value === 'vllm') void loadVllmOptions();
     loadData();
 };
 
@@ -851,7 +1045,7 @@ const refreshDashboard = async () => {
     clearDashboardCache();
     onLoadBaseInfo(false, '');
     hasRefreshedOptionsOnHover.value = false;
-    await Promise.allSettled([onLoadNetworkOptions(true), onLoadIOOptions(true), loadSettingInfo()]);
+    await Promise.allSettled([onLoadNetworkOptions(true), onLoadIOOptions(true), loadSettingInfo(), loadVllmOptions()]);
     MsgSuccess(i18n.global.t('commons.msg.refreshSuccess'));
 };
 
@@ -1028,7 +1222,7 @@ const saveMemo = async () => {
 const loadData = async () => {
     if (chartOption.value === 'io') {
         chartsOption.value['ioChart'] = loadIOChartOption();
-    } else {
+    } else if (chartOption.value === 'network') {
         chartsOption.value['networkChart'] = loadNetworkChartOption();
     }
 };
@@ -1097,12 +1291,14 @@ const loadSource = (row: any) => {
 const onFocus = () => {
     isActive.value = true;
     skipNextCurrentInfoDelta.value = true;
+    void loadVllmOptions();
 };
 const onBlur = () => {
     isActive.value = false;
 };
 
 const refreshOptionsOnHover = async () => {
+    void loadVllmOptions();
     if (hasRefreshedOptionsOnHover.value) return;
     if (!netOptionsFromCache.value && !ioOptionsFromCache.value) return;
     hasRefreshedOptionsOnHover.value = true;
@@ -1155,7 +1351,29 @@ onMounted(() => {
     loadMemo();
 });
 
+watch(
+    [currentNode, isProductPro, canViewVllm],
+    () => {
+        vllmOptionsVersion++;
+        stopVllmMonitor();
+        vllmOptions.value = [];
+        vllmInstallID.value = undefined;
+        if (chartOption.value === 'vllm') chartOption.value = 'network';
+        void loadVllmOptions();
+    },
+    { immediate: true, flush: 'sync' },
+);
+
+watch([chartOption, vllmInstallID, isActive], () => {
+    stopVllmMonitor();
+    if (chartOption.value === 'vllm' && hasVllm.value && vllmInstallID.value && isActive.value) {
+        void refreshVllmMonitor();
+    }
+});
+
 onBeforeUnmount(() => {
+    vllmOptionsVersion++;
+    stopVllmMonitor();
     window.removeEventListener('focus', onFocus);
     window.removeEventListener('blur', onBlur);
     isCurrentActive.value = false;
@@ -1189,6 +1407,16 @@ onBeforeUnmount(() => {
     margin-top: 40px;
 }
 
+.vllm-monitor-footer {
+    display: flex;
+    justify-content: space-between;
+    align-items: center;
+    gap: 12px;
+    padding: 0 20px;
+    color: var(--el-text-color-secondary);
+    font-size: 12px;
+}
+
 @media only screen and (max-width: 767px) {
     .dashboard-entrance-alert {
         padding-right: 40px;