diff --git a/rustfs/src/storage/concurrency.rs b/rustfs/src/storage/concurrency.rs index 4ab95135..33b53f81 100644 --- a/rustfs/src/storage/concurrency.rs +++ b/rustfs/src/storage/concurrency.rs @@ -251,6 +251,8 @@ struct IoLoadMetrics { recent_waits: Vec, /// Maximum samples to keep in the window max_samples: usize, + /// The earliest record index in the recent_waits vector + earliest_index: usize, /// Total wait time observed (for averaging) total_wait_ns: AtomicU64, /// Total number of observations @@ -263,6 +265,7 @@ impl IoLoadMetrics { Self { recent_waits: Vec::with_capacity(max_samples), max_samples, + earliest_index: 0, total_wait_ns: AtomicU64::new(0), observation_count: AtomicU64::new(0), } @@ -271,10 +274,12 @@ impl IoLoadMetrics { /// Record a new permit wait observation fn record(&mut self, wait: Duration) { // Add to recent waits (with eviction if full) - if self.recent_waits.len() >= self.max_samples { - self.recent_waits.remove(0); + if self.recent_waits.len() < self.max_samples { + self.recent_waits.push(wait); + } else { + self.recent_waits[self.earliest_index] = wait; + self.earliest_index = (self.earliest_index + 1) % self.max_samples; } - self.recent_waits.push(wait); // Update totals for overall statistics self.total_wait_ns.fetch_add(wait.as_nanos() as u64, Ordering::Relaxed); @@ -1867,4 +1872,154 @@ mod tests { assert!(manager.is_cached("warm2").await); assert!(manager.is_cached("warm3").await); } + + #[test] + fn test_io_load_metrics_record_not_full() { + let mut metrics = IoLoadMetrics::new(5); + metrics.record(Duration::from_millis(10)); + metrics.record(Duration::from_millis(20)); + assert_eq!(metrics.recent_waits.len(), 2); + assert_eq!(metrics.recent_waits[0], Duration::from_millis(10)); + assert_eq!(metrics.recent_waits[1], Duration::from_millis(20)); + assert_eq!(metrics.observation_count(), 2); + } + + #[test] + fn test_io_load_metrics_record_full_and_circular() { + let mut metrics = IoLoadMetrics::new(3); + metrics.record(Duration::from_millis(10)); + metrics.record(Duration::from_millis(20)); + metrics.record(Duration::from_millis(30)); + assert_eq!(metrics.recent_waits.len(), 3); + assert_eq!(metrics.earliest_index, 0); + + // This should overwrite the first element (10ms) + metrics.record(Duration::from_millis(40)); + assert_eq!(metrics.recent_waits.len(), 3); + assert_eq!(metrics.recent_waits[0], Duration::from_millis(40)); + assert_eq!(metrics.recent_waits[1], Duration::from_millis(20)); + assert_eq!(metrics.recent_waits[2], Duration::from_millis(30)); + assert_eq!(metrics.earliest_index, 1); + assert_eq!(metrics.observation_count(), 4); + + // This should overwrite the second element (20ms) + metrics.record(Duration::from_millis(50)); + assert_eq!(metrics.recent_waits.len(), 3); + assert_eq!(metrics.recent_waits[0], Duration::from_millis(40)); + assert_eq!(metrics.recent_waits[1], Duration::from_millis(50)); + assert_eq!(metrics.recent_waits[2], Duration::from_millis(30)); + assert_eq!(metrics.earliest_index, 2); + assert_eq!(metrics.observation_count(), 5); + + // This should overwrite the third element (30ms) + metrics.record(Duration::from_millis(60)); + assert_eq!(metrics.recent_waits.len(), 3); + assert_eq!(metrics.recent_waits[0], Duration::from_millis(40)); + assert_eq!(metrics.recent_waits[1], Duration::from_millis(50)); + assert_eq!(metrics.recent_waits[2], Duration::from_millis(60)); + assert_eq!(metrics.earliest_index, 0); + assert_eq!(metrics.observation_count(), 6); + } + + #[test] + fn test_io_load_metrics_average_wait() { + let mut metrics = IoLoadMetrics::new(3); + metrics.record(Duration::from_millis(10)); + metrics.record(Duration::from_millis(20)); + metrics.record(Duration::from_millis(30)); + assert_eq!(metrics.average_wait(), Duration::from_millis(20)); + + // Overwrite 10ms with 40ms, new avg = (20+30+40)/3 = 30 + metrics.record(Duration::from_millis(40)); + assert_eq!(metrics.average_wait(), Duration::from_millis(30)); + } + + #[test] + fn test_io_load_metrics_max_wait() { + let mut metrics = IoLoadMetrics::new(3); + assert_eq!(metrics.max_wait(), Duration::ZERO); + metrics.record(Duration::from_millis(40)); + metrics.record(Duration::from_millis(30)); + metrics.record(Duration::from_millis(20)); + assert_eq!(metrics.max_wait(), Duration::from_millis(40)); + + // Overwrite 40ms with 5ms, max should still be 30 + metrics.record(Duration::from_millis(5)); + assert_eq!(metrics.max_wait(), Duration::from_millis(30)); + + // Overwrite 30ms with 10ms + metrics.record(Duration::from_millis(10)); + assert_eq!(metrics.max_wait(), Duration::from_millis(20)); + } + + #[test] + fn test_io_load_metrics_p95_wait() { + let mut metrics = IoLoadMetrics::new(20); + for i in 1..=20 { + metrics.record(Duration::from_millis(i * 5)); // 5, 10, ..., 100 + } + assert_eq!(metrics.p95_wait(), Duration::from_millis(100)); + + // Test with different values + let mut metrics = IoLoadMetrics::new(10); + metrics.record(Duration::from_millis(10)); + metrics.record(Duration::from_millis(20)); + metrics.record(Duration::from_millis(30)); + metrics.record(Duration::from_millis(40)); + metrics.record(Duration::from_millis(50)); + metrics.record(Duration::from_millis(60)); + metrics.record(Duration::from_millis(70)); + metrics.record(Duration::from_millis(80)); + metrics.record(Duration::from_millis(90)); + metrics.record(Duration::from_millis(1000)); // outlier + assert_eq!(metrics.p95_wait(), Duration::from_millis(1000)); + } + + #[test] + fn test_io_load_metrics_smoothed_load_level() { + let mut metrics = IoLoadMetrics::new(3); + // Average is low + metrics.record(Duration::from_millis(5)); + metrics.record(Duration::from_millis(8)); + assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::Low); + + // Average is medium + metrics.record(Duration::from_millis(40)); // avg = (5+8+40)/3 = 17.6 -> Medium + assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::Medium); + + // Average is High + metrics.record(Duration::from_millis(100)); // avg = (8+40+100)/3 = 49.3 -> Medium + assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::Medium); + + metrics.record(Duration::from_millis(100)); // avg = (40+100+100)/3 = 80 -> High + assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::High); + + // Average is Critical + metrics.record(Duration::from_millis(300)); // avg = (100+100+300)/3 = 166.6 -> High + assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::High); + + metrics.record(Duration::from_millis(300)); // avg = (100+300+300)/3 = 233.3 -> Critical + assert_eq!(metrics.smoothed_load_level(), IoLoadLevel::Critical); + } + + #[test] + fn test_io_load_metrics_lifetime_average() { + let mut metrics = IoLoadMetrics::new(2); + metrics.record(Duration::from_millis(10)); + metrics.record(Duration::from_millis(20)); + // total = 30, count = 2, avg = 15 + assert_eq!(metrics.lifetime_average_wait(), Duration::from_millis(15)); + + metrics.record(Duration::from_millis(30)); // recent=(20, 30), but lifetime avg is over all records + // total = 10+20+30=60, count = 3, avg = 20 + let total_ns = metrics.total_wait_ns.load(Ordering::Relaxed); + let count = metrics.observation_count.load(Ordering::Relaxed); + assert_eq!(total_ns, 60_000_000); + assert_eq!(count, 3); + assert_eq!(metrics.lifetime_average_wait(), Duration::from_millis(20)); + + metrics.record(Duration::from_millis(40)); + // total = 60+40=100, count = 4, avg = 25 + assert_eq!(metrics.lifetime_average_wait(), Duration::from_millis(25)); + } }