feat: publish sanitized custom.97 source

common/system_monitor.go

@@ -1,6 +1,7 @@
 package common
 
 import (
+	"fmt"
 	"sync/atomic"
 	"time"
 
@@ -25,10 +26,36 @@ type SystemStatus struct {
 	CPUUsage    float64
 	MemoryUsage float64
 	DiskUsage   float64
+	SampledAt   time.Time
+	// 过载判定带连续性要求：单一采样尖峰不再直接熔断，需连续多次超阈值。
+	CPUOverloaded    bool
+	MemoryOverloaded bool
+	DiskOverloaded   bool
+}
+
+// SystemOverload 是哨兵据以拒绝请求的过载项，Error 文本带当前值与生效阈值。
+type SystemOverload struct {
+	Code      string
+	Resource  string
+	Usage     float64
+	Threshold int
+}
+
+func (o *SystemOverload) Error() string {
+	return fmt.Sprintf("system %s overloaded (current: %.1f%%, threshold: %d%%)", o.Resource, o.Usage, o.Threshold)
 }
 
 var latestSystemStatus atomic.Value
 
+const (
+	systemMonitorInterval = 5 * time.Second
+	// 连续 3 次采样（约 15 秒）仍高于阈值才判定过载，
+	// 任一采样回落到阈值以下即恢复，避免瞬时尖峰打满重试。
+	overloadConsecutiveSamples = 3
+	// 采样超过该时长未刷新（监控未运行或被阻塞）时，哨兵不据此熔断。
+	systemStatusMaxAge = 30 * time.Second
+)
+
 func init() {
 	latestSystemStatus.Store(SystemStatus{})
 }
@@ -36,46 +63,98 @@ func init() {
 // StartSystemMonitor 启动系统监控
 func StartSystemMonitor() {
 	go func() {
+		usage := newContainerUsage()
+		var detector overloadDetector
 		for {
 			config := GetPerformanceMonitorConfig()
 			if !config.Enabled {
-				time.Sleep(30 * time.Second)
+				detector = overloadDetector{}
+				latestSystemStatus.Store(SystemStatus{SampledAt: time.Now()})
+				time.Sleep(systemMonitorInterval)
 				continue
 			}
 
-			updateSystemStatus()
-			time.Sleep(5 * time.Second)
+			status := collectSystemStatus(usage)
+			detector.mark(&status, config)
+			latestSystemStatus.Store(status)
+
+			time.Sleep(systemMonitorInterval)
 		}
 	}()
 }
 
-func updateSystemStatus() {
-	var status SystemStatus
+// overloadDetector 为每项资源累计连续超阈值的采样次数：连续 overloadConsecutiveSamples 次
+// 才标记过载，任一次回落立即清零并重新累计。
+type overloadDetector struct {
+	cpuSamples    int
+	memorySamples int
+	diskSamples   int
+}
 
-	// CPU
-	// 注意：cpu.Percent(0, false) 返回自上次调用以来的 CPU 使用率
-	// 如果是第一次调用，可能会返回错误或不准确的值，但在循环中会逐渐正常
-	percents, err := cpu.Percent(0, false)
-	if err == nil && len(percents) > 0 {
+func (d *overloadDetector) mark(status *SystemStatus, config PerformanceMonitorConfig) {
+	status.CPUOverloaded = countOverloadSample(&d.cpuSamples, exceedsThreshold(status.CPUUsage, config.CPUThreshold))
+	status.MemoryOverloaded = countOverloadSample(&d.memorySamples, exceedsThreshold(status.MemoryUsage, config.MemoryThreshold))
+	status.DiskOverloaded = countOverloadSample(&d.diskSamples, exceedsThreshold(status.DiskUsage, config.DiskThreshold))
+}
+
+func countOverloadSample(samples *int, over bool) bool {
+	if !over {
+		*samples = 0
+		return false
+	}
+	*samples = min(*samples+1, overloadConsecutiveSamples)
+	return *samples == overloadConsecutiveSamples
+}
+
+// exceedsThreshold 判断采样值是否超过阈值；阈值 <= 0 表示不检查该项。
+func exceedsThreshold(usage float64, threshold int) bool {
+	return threshold > 0 && int(usage) > threshold
+}
+
+func collectSystemStatus(usage *containerUsage) SystemStatus {
+	now := time.Now()
+	status := SystemStatus{SampledAt: now}
+
+	// CPU / 内存优先按容器 cgroup 配额计算；容器口径不可用时回退宿主机口径，
+	// 保持对未限制资源或被挂载为宿主 cgroup 的环境的兼容。
+	if saturation, ok := usage.cpuSaturation(now); ok {
+		status.CPUUsage = saturation
+	} else if percents, err := cpu.Percent(0, false); err == nil && len(percents) > 0 {
 		status.CPUUsage = percents[0]
 	}
-
-	// Memory
-	memInfo, err := mem.VirtualMemory()
-	if err == nil {
+	if containerMemory, ok := usage.memoryUsage(); ok {
+		status.MemoryUsage = containerMemory
+	} else if memInfo, err := mem.VirtualMemory(); err == nil {
 		status.MemoryUsage = memInfo.UsedPercent
 	}
 
-	// Disk
 	diskInfo := GetDiskSpaceInfo()
 	if diskInfo.Total > 0 {
 		status.DiskUsage = diskInfo.UsedPercent
 	}
-
-	latestSystemStatus.Store(status)
+	return status
 }
 
 // GetSystemStatus 获取当前系统状态
 func GetSystemStatus() SystemStatus {
 	return latestSystemStatus.Load().(SystemStatus)
 }
+
+// CheckSystemOverload 返回哨兵应据以熔断的过载项。监控关闭、采样缺失或已超过
+// systemStatusMaxAge 时返回 nil。过载标记来自连续采样，阈值按当前配置复核，
+// 运行时关闭或调高阈值后立即放行，不必等待下一次采样。
+func CheckSystemOverload(status SystemStatus, config PerformanceMonitorConfig, now time.Time) *SystemOverload {
+	if !config.Enabled || status.SampledAt.IsZero() || now.Sub(status.SampledAt) > systemStatusMaxAge {
+		return nil
+	}
+	if status.CPUOverloaded && exceedsThreshold(status.CPUUsage, config.CPUThreshold) {
+		return &SystemOverload{Code: "system_cpu_overloaded", Resource: "cpu", Usage: status.CPUUsage, Threshold: config.CPUThreshold}
+	}
+	if status.MemoryOverloaded && exceedsThreshold(status.MemoryUsage, config.MemoryThreshold) {
+		return &SystemOverload{Code: "system_memory_overloaded", Resource: "memory", Usage: status.MemoryUsage, Threshold: config.MemoryThreshold}
+	}
+	if status.DiskOverloaded && exceedsThreshold(status.DiskUsage, config.DiskThreshold) {
+		return &SystemOverload{Code: "system_disk_overloaded", Resource: "disk", Usage: status.DiskUsage, Threshold: config.DiskThreshold}
+	}
+	return nil
+}