diff --git a/Cargo.lock b/Cargo.lock index c879160c..476264d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7695,8 +7695,8 @@ dependencies = [ "rustfs-config", "rustfs-utils", "serde", - "smallvec", "sysinfo", + "temp-env", "tempfile", "thiserror 2.0.18", "tokio", @@ -9658,9 +9658,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", diff --git a/Cargo.toml b/Cargo.toml index 29c6ba03..c6741465 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -258,7 +258,7 @@ tracing = { version = "0.1.44" } tracing-appender = "0.2.4" tracing-error = "0.2.1" tracing-opentelemetry = "0.32.1" -tracing-subscriber = { version = "0.3.22", features = ["env-filter", "time"] } +tracing-subscriber = { version = "0.3.23", features = ["env-filter", "time"] } transform-stream = "0.3.1" url = "2.5.8" urlencoding = "2.1.3" diff --git a/crates/config/src/observability/mod.rs b/crates/config/src/observability/mod.rs index 6b90dad6..46cef701 100644 --- a/crates/config/src/observability/mod.rs +++ b/crates/config/src/observability/mod.rs @@ -40,7 +40,6 @@ pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL"; pub const ENV_OBS_LOG_STDOUT_ENABLED: &str = "RUSTFS_OBS_LOG_STDOUT_ENABLED"; pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY"; pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME"; -pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB"; pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME"; pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES"; @@ -64,7 +63,7 @@ pub const DEFAULT_OBS_LOG_COMPRESS_OLD_FILES: bool = true; pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_LEVEL: u32 = 6; pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_EXTENSION: &str = "gz"; pub const DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION: &str = concat!(".", DEFAULT_OBS_LOG_GZIP_COMPRESSION_EXTENSION); -pub const DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS: u64 = 30; +pub const DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS: u64 = 30; // Retain compressed files for 30 days pub const DEFAULT_OBS_LOG_DELETE_EMPTY_FILES: bool = true; pub const DEFAULT_OBS_LOG_MIN_FILE_AGE_SECONDS: u64 = 3600; // 1 hour pub const DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS: u64 = 1800; // 0.5 hours @@ -103,7 +102,6 @@ mod tests { assert_eq!(ENV_OBS_LOG_STDOUT_ENABLED, "RUSTFS_OBS_LOG_STDOUT_ENABLED"); assert_eq!(ENV_OBS_LOG_DIRECTORY, "RUSTFS_OBS_LOG_DIRECTORY"); assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME"); - assert_eq!(ENV_OBS_LOG_ROTATION_SIZE_MB, "RUSTFS_OBS_LOG_ROTATION_SIZE_MB"); assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME"); assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES"); assert_eq!(ENV_OBS_TRACES_EXPORT_ENABLED, "RUSTFS_OBS_TRACES_EXPORT_ENABLED"); diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index dc4a7f7d..cb8310f2 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -48,7 +48,6 @@ opentelemetry-stdout = { workspace = true } opentelemetry-otlp = { workspace = true } opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] } serde = { workspace = true } -smallvec = { workspace = true, features = ["serde"] } tracing = { workspace = true, features = ["std", "attributes"] } tracing-appender = { workspace = true } tracing-error = { workspace = true } @@ -65,3 +64,4 @@ pyroscope = { workspace = true, features = ["backend-pprof-rs"] } [dev-dependencies] tokio = { workspace = true, features = ["full"] } tempfile = { workspace = true } +temp-env = { workspace = true } diff --git a/crates/obs/README.md b/crates/obs/README.md index 5d38290e..13c6a456 100644 --- a/crates/obs/README.md +++ b/crates/obs/README.md @@ -7,15 +7,15 @@ logging, distributed tracing, metrics via OpenTelemetry, and continuous profilin ## Features -| Feature | Description | -|---------|-------------| -| **Structured logging** | JSON-formatted logs via `tracing-subscriber` | -| **Rolling-file logging** | Daily / hourly rotation with automatic cleanup and high-precision timestamps | -| **Distributed tracing** | OTLP/HTTP export to Jaeger, Tempo, or any OTel collector | -| **Metrics** | OTLP/HTTP export, bridged from the `metrics` crate facade | -| **Continuous Profiling** | CPU/Memory profiling export to Pyroscope | -| **Log cleanup** | Background task: size limits, gzip compression, retention policies | -| **GPU metrics** *(optional)* | Enable with the `gpu` feature flag | +| Feature | Description | +|------------------------------|------------------------------------------------------------------------------| +| **Structured logging** | JSON-formatted logs via `tracing-subscriber` | +| **Rolling-file logging** | Daily / hourly rotation with automatic cleanup and high-precision timestamps | +| **Distributed tracing** | OTLP/HTTP export to Jaeger, Tempo, or any OTel collector | +| **Metrics** | OTLP/HTTP export, bridged from the `metrics` crate facade | +| **Continuous Profiling** | CPU/Memory profiling export to Pyroscope | +| **Log cleanup** | Background task: size limits, gzip compression, retention policies | +| **GPU metrics** *(optional)* | Enable with the `gpu` feature flag | --- @@ -44,7 +44,7 @@ async fn main() { } ``` -> **Keep `_guard` alive** for the lifetime of your application. Dropping it +> **Keep `_guard` alive** for the lifetime of your application. Dropping it > triggers an ordered shutdown of every OpenTelemetry provider. --- @@ -57,8 +57,8 @@ async fn main() { use rustfs_obs::init_obs; let _guard = init_obs(Some("http://otel-collector:4318".to_string())) - .await - .expect("observability init failed"); +.await +.expect("observability init failed"); ``` ### With a custom config struct @@ -67,9 +67,9 @@ let _guard = init_obs(Some("http://otel-collector:4318".to_string())) use rustfs_obs::{AppConfig, OtelConfig, init_obs_with_config}; let config = AppConfig::new_with_endpoint(Some("http://localhost:4318".to_string())); -let _guard = init_obs_with_config(&config.observability) - .await - .expect("observability init failed"); +let _guard = init_obs_with_config( & config.observability) +.await +.expect("observability init failed"); ``` --- @@ -92,10 +92,12 @@ The library selects a backend automatically based on configuration: ``` **Key Points:** + - When **no log directory** is configured, logs automatically go to **stdout only** (perfect for development) - When a **log directory** is set, logs go to **rolling files** in that directory - In **non-production environments**, stdout is automatically mirrored alongside file logging for visibility -- In **production** mode, you must explicitly set `RUSTFS_OBS_LOG_STDOUT_ENABLED=true` to see stdout in addition to files +- In **production** mode, you must explicitly set `RUSTFS_OBS_LOG_STDOUT_ENABLED=true` to see stdout in addition to + files --- @@ -105,56 +107,55 @@ All configuration is read from environment variables at startup. ### OTLP / Export -| Variable | Default | Description | -|----------|---------|-------------| -| `RUSTFS_OBS_ENDPOINT` | _(empty)_ | Root OTLP/HTTP endpoint, e.g. `http://otel-collector:4318` | -| `RUSTFS_OBS_TRACE_ENDPOINT` | _(empty)_ | Dedicated trace endpoint (overrides root + `/v1/traces`) | -| `RUSTFS_OBS_METRIC_ENDPOINT` | _(empty)_ | Dedicated metrics endpoint | -| `RUSTFS_OBS_LOG_ENDPOINT` | _(empty)_ | Dedicated log endpoint | -| `RUSTFS_OBS_PROFILING_ENDPOINT` | _(empty)_ | Dedicated profiling endpoint (e.g. Pyroscope) | -| `RUSTFS_OBS_TRACES_EXPORT_ENABLED` | `true` | Toggle trace export | -| `RUSTFS_OBS_METRICS_EXPORT_ENABLED` | `true` | Toggle metrics export | -| `RUSTFS_OBS_LOGS_EXPORT_ENABLED` | `true` | Toggle OTLP log export | -| `RUSTFS_OBS_PROFILING_EXPORT_ENABLED` | `true` | Toggle profiling export | -| `RUSTFS_OBS_USE_STDOUT` | `false` | Mirror all signals to stdout alongside OTLP | -| `RUSTFS_OBS_SAMPLE_RATIO` | `0.1` | Trace sampling ratio `0.0`–`1.0` | -| `RUSTFS_OBS_METER_INTERVAL` | `15` | Metrics export interval (seconds) | +| Variable | Default | Description | +|---------------------------------------|-----------|------------------------------------------------------------| +| `RUSTFS_OBS_ENDPOINT` | _(empty)_ | Root OTLP/HTTP endpoint, e.g. `http://otel-collector:4318` | +| `RUSTFS_OBS_TRACE_ENDPOINT` | _(empty)_ | Dedicated trace endpoint (overrides root + `/v1/traces`) | +| `RUSTFS_OBS_METRIC_ENDPOINT` | _(empty)_ | Dedicated metrics endpoint | +| `RUSTFS_OBS_LOG_ENDPOINT` | _(empty)_ | Dedicated log endpoint | +| `RUSTFS_OBS_PROFILING_ENDPOINT` | _(empty)_ | Dedicated profiling endpoint (e.g. Pyroscope) | +| `RUSTFS_OBS_TRACES_EXPORT_ENABLED` | `true` | Toggle trace export | +| `RUSTFS_OBS_METRICS_EXPORT_ENABLED` | `true` | Toggle metrics export | +| `RUSTFS_OBS_LOGS_EXPORT_ENABLED` | `true` | Toggle OTLP log export | +| `RUSTFS_OBS_PROFILING_EXPORT_ENABLED` | `true` | Toggle profiling export | +| `RUSTFS_OBS_USE_STDOUT` | `false` | Mirror all signals to stdout alongside OTLP | +| `RUSTFS_OBS_SAMPLE_RATIO` | `0.1` | Trace sampling ratio `0.0`–`1.0` | +| `RUSTFS_OBS_METER_INTERVAL` | `15` | Metrics export interval (seconds) | ### Service identity -| Variable | Default | Description | -|----------|---------|-------------| -| `RUSTFS_OBS_SERVICE_NAME` | `rustfs` | OTel `service.name` | -| `RUSTFS_OBS_SERVICE_VERSION` | _(crate version)_ | OTel `service.version` | -| `RUSTFS_OBS_ENVIRONMENT` | `development` | Deployment environment (`production`, `development`, …) | +| Variable | Default | Description | +|------------------------------|-------------------|---------------------------------------------------------| +| `RUSTFS_OBS_SERVICE_NAME` | `rustfs` | OTel `service.name` | +| `RUSTFS_OBS_SERVICE_VERSION` | _(crate version)_ | OTel `service.version` | +| `RUSTFS_OBS_ENVIRONMENT` | `development` | Deployment environment (`production`, `development`, …) | ### Local logging -| Variable | Default | Description | -|----------|---------|-------------| -| `RUSTFS_OBS_LOGGER_LEVEL` | `info` | Log level; `RUST_LOG` syntax supported | -| `RUSTFS_OBS_LOG_STDOUT_ENABLED` | `false` | When file logging is active, also mirror to stdout | -| `RUSTFS_OBS_LOG_DIRECTORY` | _(empty)_ | **Directory for rolling log files. When empty, logs go to stdout only** | -| `RUSTFS_OBS_LOG_FILENAME` | `rustfs.log` | Base filename for rolling logs. Rotated archives include a high-precision timestamp and counter. With the default `RUSTFS_OBS_LOG_MATCH_MODE=suffix`, names look like `-.rustfs.log` (e.g., `20231027103001.123456-0.rustfs.log`); with `prefix`, they look like `rustfs.log.-` (e.g., `rustfs.log.20231027103001.123456-0`). | -| `RUSTFS_OBS_LOG_ROTATION_TIME` | `hourly` | Rotation granularity: `minutely`, `hourly`, or `daily` | -| `RUSTFS_OBS_LOG_KEEP_FILES` | `30` | Number of rolling files to keep (also used by cleaner) | -| `RUSTFS_OBS_LOG_MATCH_MODE` | `suffix` | File matching mode: `prefix` or `suffix` | +| Variable | Default | Description | +|---------------------------------|--------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| `RUSTFS_OBS_LOGGER_LEVEL` | `info` | Log level; `RUST_LOG` syntax supported | +| `RUSTFS_OBS_LOG_STDOUT_ENABLED` | `false` | When file logging is active, also mirror to stdout | +| `RUSTFS_OBS_LOG_DIRECTORY` | _(empty)_ | **Directory for rolling log files. When empty, logs go to stdout only** | +| `RUSTFS_OBS_LOG_FILENAME` | `rustfs.log` | Base filename for rolling logs. Rotated archives include a high-precision timestamp and counter. With the default `RUSTFS_OBS_LOG_MATCH_MODE=suffix`, names look like `-.rustfs.log` (e.g., `20231027103001.123456-0.rustfs.log`); with `prefix`, they look like `rustfs.log.-` (e.g., `rustfs.log.20231027103001.123456-0`). | +| `RUSTFS_OBS_LOG_ROTATION_TIME` | `hourly` | Rotation granularity: `minutely`, `hourly`, or `daily` | +| `RUSTFS_OBS_LOG_KEEP_FILES` | `30` | Number of rolling files to keep (also used by cleaner) | +| `RUSTFS_OBS_LOG_MATCH_MODE` | `suffix` | File matching mode: `prefix` or `suffix` | ### Log cleanup -| Variable | Default | Description | -|----------|---------|-------------| -| `RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES` | `2147483648` | Hard cap on total log directory size (2 GiB) | -| `RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES` | `0` | Per-file size cap; `0` = unlimited | -| `RUSTFS_OBS_LOG_COMPRESS_OLD_FILES` | `true` | Gzip-compress files before deleting | -| `RUSTFS_OBS_LOG_GZIP_COMPRESSION_LEVEL` | `6` | Gzip level `1` (fastest) – `9` (best) | -| `RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS` | `30` | Delete `.gz` archives older than N days; `0` = keep forever | -| `RUSTFS_OBS_LOG_EXCLUDE_PATTERNS` | _(empty)_ | Comma-separated glob patterns to never clean up | -| `RUSTFS_OBS_LOG_DELETE_EMPTY_FILES` | `true` | Remove zero-byte files | -| `RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS` | `3600` | Minimum file age (seconds) before cleanup | -| `RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS` | `1800` | How often the cleanup task runs (0.5 hours) | -| `RUSTFS_OBS_LOG_DRY_RUN` | `false` | Report deletions without actually removing files | - +| Variable | Default | Description | +|-------------------------------------------------|--------------|-------------------------------------------------------------| +| `RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES` | `2147483648` | Hard cap on total log directory size (2 GiB) | +| `RUSTFS_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES` | `0` | Per-file size cap; `0` = unlimited | +| `RUSTFS_OBS_LOG_COMPRESS_OLD_FILES` | `true` | Gzip-compress files before deleting | +| `RUSTFS_OBS_LOG_GZIP_COMPRESSION_LEVEL` | `6` | Gzip level `1` (fastest) – `9` (best) | +| `RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS` | `30` | Delete `.gz` archives older than N days; `0` = keep forever | +| `RUSTFS_OBS_LOG_EXCLUDE_PATTERNS` | _(empty)_ | Comma-separated glob patterns to never clean up | +| `RUSTFS_OBS_LOG_DELETE_EMPTY_FILES` | `true` | Remove zero-byte files | +| `RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS` | `3600` | Minimum file age (seconds) before cleanup | +| `RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS` | `1800` | How often the cleanup task runs (0.5 hours) | +| `RUSTFS_OBS_LOG_DRY_RUN` | `false` | Report deletions without actually removing files | --- @@ -251,9 +252,9 @@ use rustfs_obs::LogCleaner; use rustfs_obs::types::FileMatchMode; let cleaner = LogCleaner::builder( - PathBuf::from("/var/log/rustfs"), - "rustfs.log.".to_string(), // file_pattern - "rustfs.log".to_string(), // active_filename +PathBuf::from("/var/log/rustfs"), +"rustfs.log.".to_string(), // file_pattern +"rustfs.log".to_string(), // active_filename ) .match_mode(FileMatchMode::Prefix) .keep_files(10) @@ -261,7 +262,7 @@ let cleaner = LogCleaner::builder( .max_single_file_size_bytes(0) // unlimited .compress_old_files(true) .gzip_compression_level(6) -.compressed_file_retention_days(30) +.compressed_file_retention_days(7) .exclude_patterns(vec!["current.log".to_string()]) .delete_empty_files(true) .min_file_age_seconds(3600) // 1 hour @@ -276,11 +277,11 @@ println!("Deleted {deleted} files, freed {freed_bytes} bytes"); ## Feature Flags -| Flag | Description | -|------|-------------| +| Flag | Description | +|-------------|------------------------------------| | _(default)_ | Core logging, tracing, and metrics | -| `gpu` | GPU utilisation metrics via `nvml` | -| `full` | All features enabled | +| `gpu` | GPU utilisation metrics via `nvml` | +| `full` | All features enabled | ```toml # Enable GPU monitoring diff --git a/crates/obs/src/telemetry/filter.rs b/crates/obs/src/telemetry/filter.rs index 3211baec..cdb8789c 100644 --- a/crates/obs/src/telemetry/filter.rs +++ b/crates/obs/src/telemetry/filter.rs @@ -17,8 +17,8 @@ //! This module provides helper functions for building `EnvFilter` instances //! used across different logging backends (stdout, file, OpenTelemetry). -use smallvec::SmallVec; -use tracing_subscriber::EnvFilter; +use std::env; +use tracing_subscriber::{EnvFilter, filter::LevelFilter}; /// Build an `EnvFilter` from the given log level string. /// @@ -37,7 +37,8 @@ use tracing_subscriber::EnvFilter; /// # Returns /// A configured `EnvFilter` ready to be attached to a `tracing_subscriber` registry. fn is_verbose_level(level: &str) -> bool { - matches!(level.to_ascii_lowercase().as_str(), "trace" | "debug") + let s = level.trim().to_ascii_lowercase(); + s.contains("trace") || s.contains("debug") } fn rust_log_requests_verbose(rust_log: &str) -> bool { @@ -47,8 +48,14 @@ fn rust_log_requests_verbose(rust_log: &str) -> bool { return false; } - let level = directive.rsplit('=').next().unwrap_or("").trim(); - is_verbose_level(level) + // If the directive is just a level (e.g. "debug"), check it. + // If it's a module=level (e.g. "my_crate=debug"), we split by '='. + // Note: rsplit keeps the last part as the first element of iterator if we take next(). + if let Some(level_part) = directive.rsplit('=').next() { + is_verbose_level(level_part) + } else { + false + } }) } @@ -58,6 +65,13 @@ fn should_suppress_noisy_crates(logger_level: &str, default_level: Option<&str>, } if let Some(rust_log) = rust_log { + // If RUST_LOG is present, we check if ANY part of it requests verbose logging. + // If the user explicitly asks for debug/trace anywhere, we assume they are debugging + // and might want to see third-party logs unless they silence them. + // HOWEVER, standard practice is: if RUST_LOG is set, we respect it fully and + // adding extra suppressions might be confusing. + // But the original logic was: "For non-verbose levels... noisy crates are silenced". + // So if RUST_LOG="info", we suppress. If RUST_LOG="debug", we don't. return !rust_log_requests_verbose(rust_log); } @@ -65,32 +79,53 @@ fn should_suppress_noisy_crates(logger_level: &str, default_level: Option<&str>, } pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>) -> EnvFilter { - let level = default_level.unwrap_or(logger_level); - let rust_log = if default_level.is_none() { - std::env::var("RUST_LOG").ok() - } else { - None - }; - let mut filter = default_level - .map(EnvFilter::new) - .unwrap_or_else(|| EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level))); + // 1. Determine the base filter source. + // If `default_level` is set (e.g. forced override), we use it. + // Otherwise, we look at `RUST_LOG`. + // If `RUST_LOG` is missing, we fallback to `logger_level` (from config/env var `RUSTFS_OBS_LOGGER_LEVEL`). - if should_suppress_noisy_crates(logger_level, default_level, rust_log.as_deref()) { - let directives: SmallVec<[(&str, &str); 6]> = smallvec::smallvec![ - ("hyper", "off"), - ("tonic", "off"), - ("h2", "off"), - ("reqwest", "off"), - ("tower", "off"), + let rust_log_env = env::var("RUST_LOG").ok(); + + // Logic: + // - If default_level is Some, use EnvFilter::new(default_level). + // - Else if RUST_LOG is set, use EnvFilter::new(rust_log). + // - Else use EnvFilter::new(logger_level). + + let mut filter = if let Some(lvl) = default_level { + EnvFilter::new(lvl) + } else if let Some(ref rust_log) = rust_log_env { + EnvFilter::new(rust_log) + } else { + EnvFilter::new(logger_level) + }; + + // 2. Apply noisy crate suppression if needed. + // We only suppress if the effective configuration is NOT verbose (i.e. not debug/trace). + if should_suppress_noisy_crates(logger_level, default_level, rust_log_env.as_deref()) { + let directives = [ + ("hyper", LevelFilter::OFF), + ("tonic", LevelFilter::OFF), + ("h2", LevelFilter::OFF), + ("reqwest", LevelFilter::OFF), + ("tower", LevelFilter::OFF), // HTTP request logs are demoted to WARN to reduce volume in production. - ("rustfs::server::http", "warn"), + ("rustfs::server::http", LevelFilter::WARN), ]; + for (crate_name, level) in directives { + // We use `add_directive` which effectively appends to the filter. + // If RUST_LOG already specified `hyper=debug`, adding `hyper=off` later MIGHT override it + // depending on specificity, but usually the last directive wins or the most specific one. + // EnvFilter parsing order matters. + + // To be safe and respectful of RUST_LOG, we should arguably NOT override if RUST_LOG is set? + // BUT, the requirement says: "For non-verbose levels... noisy internal crates... are silenced". + // So if RUST_LOG="info", we DO want to silence hyper. + // If RUST_LOG="hyper=debug", `should_suppress_noisy_crates` returns false, so we won't silence it. + match format!("{crate_name}={level}").parse() { Ok(directive) => filter = filter.add_directive(directive), Err(e) => { - // The directive strings are compile-time constants, so this - // branch should never be reached; emit a diagnostic just in case. eprintln!("obs: invalid log filter directive '{crate_name}={level}': {e}"); } } @@ -105,46 +140,95 @@ mod tests { use super::*; #[test] - fn test_build_env_filter_default_level_overrides() { - // Ensure that providing a default_level uses it instead of logger_level. - let filter = build_env_filter("debug", Some("error")); - // The Debug output uses `LevelFilter::ERROR` for the error level directive. - let dbg = format!("{filter:?}"); - assert!( - dbg.contains("LevelFilter::ERROR"), - "Expected 'LevelFilter::ERROR' in filter debug output: {dbg}" - ); + fn test_is_verbose_level() { + assert!(is_verbose_level("debug")); + assert!(is_verbose_level("trace")); + assert!(is_verbose_level("DEBUG")); + assert!(is_verbose_level("info,rustfs=debug")); + assert!(!is_verbose_level("info")); + assert!(!is_verbose_level("warn")); } #[test] - fn test_build_env_filter_suppresses_noisy_crates() { - // For info level, hyper/tonic/etc. should be suppressed with OFF. - let filter = build_env_filter("debug", Some("info")); - let dbg = format!("{filter:?}"); - // The Debug output uses `LevelFilter::OFF` for suppressed crates. - assert!( - dbg.contains("LevelFilter::OFF"), - "Expected 'LevelFilter::OFF' suppression directives in filter: {dbg}" - ); + fn test_rust_log_requests_verbose() { + assert!(rust_log_requests_verbose("debug")); + assert!(rust_log_requests_verbose("rustfs=debug")); + assert!(rust_log_requests_verbose("info,rustfs=trace")); + assert!(!rust_log_requests_verbose("info")); + assert!(!rust_log_requests_verbose("rustfs=info")); } #[test] - fn test_build_env_filter_debug_no_suppression() { - // For debug level, our code does NOT inject any OFF directives. - let filter = build_env_filter("info", Some("debug")); - let dbg = format!("{filter:?}"); - // Verify the filter builds without panicking and contains the debug level. - assert!(!dbg.is_empty()); - assert!( - dbg.contains("LevelFilter::DEBUG"), - "Expected 'LevelFilter::DEBUG' in filter debug output: {dbg}" - ); - } + fn test_should_suppress() { + // Case 1: logger_level="info", no RUST_LOG -> suppress + assert!(should_suppress_noisy_crates("info", None, None)); - #[test] - fn test_should_suppress_noisy_crates_respects_rust_log_debug() { + // Case 2: logger_level="debug", no RUST_LOG -> no suppress + assert!(!should_suppress_noisy_crates("debug", None, None)); + + // Case 3: RUST_LOG="info" -> suppress + assert!(should_suppress_noisy_crates("debug", None, Some("info"))); + + // Case 4: RUST_LOG="debug" -> no suppress assert!(!should_suppress_noisy_crates("info", None, Some("debug"))); - assert!(!should_suppress_noisy_crates("info", None, Some("s3=info,hyper=debug"))); - assert!(should_suppress_noisy_crates("info", None, Some("info"))); + + // Case 5: RUST_LOG="rustfs=debug" -> no suppress + assert!(!should_suppress_noisy_crates("info", None, Some("rustfs=debug"))); + } + + #[test] + fn test_build_env_filter_injects_suppressions_without_rust_log() { + // When RUST_LOG is not set and the base level is non-verbose ("info"), + // build_env_filter should inject suppression directives for noisy crates. + temp_env::with_var("RUST_LOG", None::<&str>, || { + let filter = build_env_filter("info", None); + let filter_str = filter.to_string(); + + for noisy_crate in ["hyper", "tonic", "h2", "reqwest", "tower"] { + assert!( + filter_str.contains(noisy_crate), + "expected EnvFilter to contain suppression directive for `{}`; got `{}`", + noisy_crate, + filter_str + ); + } + }); + } + + #[test] + fn test_build_env_filter_respects_verbose_rust_log() { + // When RUST_LOG requests a verbose level, automatic noisy-crate suppression + // should not be applied, even if the logger_level is non-verbose. + temp_env::with_var("RUST_LOG", Some("debug"), || { + let filter = build_env_filter("info", None); + let filter_str = filter.to_string(); + + // We assume "off" is used to silence noisy crates; absence of these + // directives indicates that suppression was not injected. + for noisy_crate in ["hyper", "tonic", "h2", "reqwest", "tower"] { + let directive = format!("{}=off", noisy_crate); + assert!( + !filter_str.contains(&directive), + "did not expect EnvFilter to contain `{}` when RUST_LOG is verbose; got `{}`", + directive, + filter_str + ); + } + }); + } + + #[test] + fn test_build_env_filter_precedence_of_rust_log_over_logger_level() { + // When default_level is None, RUST_LOG should override logger_level. + temp_env::with_var("RUST_LOG", Some("warn"), || { + let filter = build_env_filter("debug", None); + let filter_str = filter.to_string(); + + assert!( + filter_str.contains("warn"), + "expected EnvFilter to reflect RUST_LOG=warn; got `{}`", + filter_str + ); + }); } } diff --git a/crates/obs/src/telemetry/local.rs b/crates/obs/src/telemetry/local.rs index dec0f4b1..a3d45efd 100644 --- a/crates/obs/src/telemetry/local.rs +++ b/crates/obs/src/telemetry/local.rs @@ -286,7 +286,7 @@ fn init_file_logging_internal( /// This prevents world-writable log directories from being a security hazard. /// No-ops if permissions are already `0755` or stricter. #[cfg(unix)] -fn ensure_dir_permissions(log_directory: &str) -> Result<(), TelemetryError> { +pub(super) fn ensure_dir_permissions(log_directory: &str) -> Result<(), TelemetryError> { use std::fs::Permissions; use std::os::unix::fs::PermissionsExt; @@ -335,7 +335,7 @@ fn ensure_dir_permissions(log_directory: &str) -> Result<(), TelemetryError> { /// /// # Returns /// A [`tokio::task::JoinHandle`] for the spawned cleanup loop. -fn spawn_cleanup_task( +pub(super) fn spawn_cleanup_task( config: &OtelConfig, log_directory: &str, log_filename: &str, diff --git a/crates/obs/src/telemetry/mod.rs b/crates/obs/src/telemetry/mod.rs index 7ac8aff0..f7f0a8e9 100644 --- a/crates/obs/src/telemetry/mod.rs +++ b/crates/obs/src/telemetry/mod.rs @@ -76,7 +76,7 @@ use rustfs_utils::get_env_opt_str; /// application. Dropping it triggers ordered shutdown of all providers. /// /// # Errors -/// Returns [`TelemetryError`] when a backend fails to initialise (e.g., cannot +/// Returns [`TelemetryError`] when a backend fails to initialize (e.g., cannot /// create the log directory, or an OTLP exporter cannot connect). pub(crate) fn init_telemetry(config: &OtelConfig) -> Result { let environment = config.environment.as_deref().unwrap_or(ENVIRONMENT); diff --git a/crates/obs/src/telemetry/otel.rs b/crates/obs/src/telemetry/otel.rs index 620ca85d..8797683c 100644 --- a/crates/obs/src/telemetry/otel.rs +++ b/crates/obs/src/telemetry/otel.rs @@ -33,12 +33,14 @@ //! compression for efficiency over the wire. use crate::TelemetryError; +use crate::cleaner::types::FileMatchMode; use crate::config::OtelConfig; use crate::global::OBSERVABILITY_METRIC_ENABLED; use crate::telemetry::filter::build_env_filter; use crate::telemetry::guard::OtelGuard; use crate::telemetry::recorder::Recorder; use crate::telemetry::resource::build_resource; +use crate::telemetry::rolling::{RollingAppender, Rotation}; use metrics::counter; use opentelemetry::{global, trace::TracerProvider}; use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; @@ -49,11 +51,12 @@ use opentelemetry_sdk::{ metrics::{PeriodicReader, SdkMeterProvider}, trace::{RandomIdGenerator, Sampler, SdkTracerProvider}, }; +use rustfs_config::observability::DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES; use rustfs_config::{ - APP_NAME, DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED, DEFAULT_OBS_METRICS_EXPORT_ENABLED, - DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO, + APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_ROTATION_TIME, DEFAULT_OBS_LOG_STDOUT_ENABLED, DEFAULT_OBS_LOGS_EXPORT_ENABLED, + DEFAULT_OBS_METRICS_EXPORT_ENABLED, DEFAULT_OBS_TRACES_EXPORT_ENABLED, METER_INTERVAL, SAMPLE_RATIO, }; -use std::{io::IsTerminal, time::Duration}; +use std::{fs, io::IsTerminal, time::Duration}; use tracing::info; use tracing_error::ErrorLayer; use tracing_opentelemetry::{MetricsLayer, OpenTelemetryLayer}; @@ -64,6 +67,9 @@ use tracing_subscriber::{ util::SubscriberInitExt, }; +// Import helper functions from local.rs (sibling module) +use super::local::{ensure_dir_permissions, spawn_cleanup_task}; + /// Initialize the full OpenTelemetry HTTP pipeline (traces + metrics + logs). /// /// This function is invoked when at least one OTLP endpoint has been @@ -108,21 +114,41 @@ pub(super) fn init_observability_http( .as_deref() .filter(|s| !s.is_empty()) .map(|s| s.to_string()) - .unwrap_or_else(|| format!("{root_ep}/v1/traces")); + .unwrap_or_else(|| { + if !root_ep.is_empty() { + format!("{root_ep}/v1/traces") + } else { + String::new() + } + }); let metric_ep: String = config .metric_endpoint .as_deref() .filter(|s| !s.is_empty()) .map(|s| s.to_string()) - .unwrap_or_else(|| format!("{root_ep}/v1/metrics")); + .unwrap_or_else(|| { + if !root_ep.is_empty() { + format!("{root_ep}/v1/metrics") + } else { + String::new() + } + }); + // If log_endpoint is not explicitly set, fallback to root_ep/v1/logs ONLY if root_ep is present. + // If both are empty, log_ep is empty, which triggers the fallback to file logging logic. let log_ep: String = config .log_endpoint .as_deref() .filter(|s| !s.is_empty()) .map(|s| s.to_string()) - .unwrap_or_else(|| format!("{root_ep}/v1/logs")); + .unwrap_or_else(|| { + if !root_ep.is_empty() { + format!("{root_ep}/v1/logs") + } else { + String::new() + } + }); // ── Tracer provider (HTTP) ──────────────────────────────────────────────── let tracer_provider = build_tracer_provider(&trace_ep, config, res.clone(), sampler, use_stdout)?; @@ -130,48 +156,132 @@ pub(super) fn init_observability_http( // ── Meter provider (HTTP) ───────────────────────────────────────────────── let meter_provider = build_meter_provider(&metric_ep, config, res.clone(), &service_name, use_stdout)?; - // ── Logger provider (HTTP) ──────────────────────────────────────────────── - let logger_provider = build_logger_provider(&log_ep, config, res, use_stdout)?; - #[cfg(unix)] let profiling_agent = init_profiler(config); + // ── Logger Logic ────────────────────────────────────────────────────────── + let mut logger_provider: Option = None; + let mut otel_bridge = None; + let mut file_layer_opt = None; // File layer (File mode) + let mut stdout_layer_opt = None; // Stdout layer (File mode) + let mut cleanup_handle = None; + let mut tracing_guard = None; // Guard for file writer + let mut stdout_guard = None; // Guard for stdout writer (File mode) + + // ── Case 1: OTLP Logging + if !log_ep.is_empty() { + // Init OTLP logger logic. + // We initialize the OTLP collector and honor the configured stdout setting + // (e.g. via RUSTFS_OBS_USE_STDOUT / config.use_stdout) when building the provider. + logger_provider = build_logger_provider(&log_ep, config, res, false)?; + + // Build bridge to capture `tracing` events. + otel_bridge = logger_provider + .as_ref() + .map(|p| OpenTelemetryTracingBridge::new(p).with_filter(build_env_filter(logger_level, None))); + + // Note: We do NOT create a separate `fmt_layer_opt` here; stdout behavior is driven by the provider. + } + let span_events = if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL }; + // ── Case 2: File Logging + // Supplement: If log_directory is set and no OTLP log endpoint is configured, we enable file logging logic. + if let Some(log_directory) = config.log_directory.as_deref().filter(|s| !s.is_empty()) + && logger_provider.is_none() + { + let log_filename = config.log_filename.as_deref().unwrap_or(&service_name); + let keep_files = config.log_keep_files.unwrap_or(DEFAULT_LOG_KEEP_FILES); + + // 1. Ensure dir exists + if let Err(e) = fs::create_dir_all(log_directory) { + return Err(TelemetryError::Io(e.to_string())); + } + // 2. Permissions + #[cfg(unix)] + ensure_dir_permissions(log_directory)?; + + // 3. Rotation + let rotation_str = config + .log_rotation_time + .as_deref() + .unwrap_or(DEFAULT_LOG_ROTATION_TIME) + .to_lowercase(); + let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() { + Some("prefix") => FileMatchMode::Prefix, + _ => FileMatchMode::Suffix, + }; + let rotation = match rotation_str.as_str() { + "minutely" => Rotation::Minutely, + "hourly" => Rotation::Hourly, + "daily" => Rotation::Daily, + _ => Rotation::Daily, + }; + let max_single_file_size = config + .log_max_single_file_size_bytes + .unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES); + + let file_appender = + RollingAppender::new(log_directory, log_filename.to_string(), rotation, max_single_file_size, match_mode)?; + + let (non_blocking, guard) = tracing_appender::non_blocking(file_appender); + tracing_guard = Some(guard); + + file_layer_opt = Some( + tracing_subscriber::fmt::layer() + .with_timer(LocalTime::rfc_3339()) + .with_target(true) + .with_ansi(false) + .with_thread_names(true) + .with_thread_ids(true) + .with_file(true) + .with_line_number(true) + .with_writer(non_blocking) + .json() + .with_current_span(true) + .with_span_list(true) + .with_span_events(span_events.clone()) + .with_filter(build_env_filter(logger_level, None)), + ); + + // Cleanup task + cleanup_handle = Some(spawn_cleanup_task(config, log_directory, log_filename, keep_files)); + + info!( + "Init file logging at '{}', rotation: {}, keep {} files", + log_directory, rotation_str, keep_files + ); + } + // ── Tracing subscriber registry ─────────────────────────────────────────── - // Build an optional stdout formatting layer. When `log_stdout_enabled` is - // false the field is `None` and tracing-subscriber will skip it. - let fmt_layer_opt = if config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) { - let enable_color = std::io::stdout().is_terminal(); - let span_event = if is_production { FmtSpan::CLOSE } else { FmtSpan::FULL }; - let layer = tracing_subscriber::fmt::layer() - .with_timer(LocalTime::rfc_3339()) - .with_target(true) - .with_ansi(enable_color) - .with_thread_names(true) - .with_thread_ids(true) - .with_file(true) - .with_line_number(true) - .json() - .with_current_span(true) - .with_span_list(true) - .with_span_events(span_event) - .with_filter(build_env_filter(logger_level, None)); - Some(layer) - } else { - None - }; - let filter = build_env_filter(logger_level, None); - let otel_bridge = logger_provider - .as_ref() - .map(|p| OpenTelemetryTracingBridge::new(p).with_filter(build_env_filter(logger_level, None))); let tracer_layer = tracer_provider .as_ref() .map(|p| OpenTelemetryLayer::new(p.tracer(service_name.to_string()))); let metrics_layer = meter_provider.as_ref().map(|p| MetricsLayer::new(p.clone())); + // Optional stdout mirror (matching init_file_logging_internal logic) + // This is separate from OTLP stdout logic. If file logging is enabled, we honor its stdout rules. + if config.log_stdout_enabled.unwrap_or(DEFAULT_OBS_LOG_STDOUT_ENABLED) || !is_production { + let (stdout_nb, stdout_g) = tracing_appender::non_blocking(std::io::stdout()); + stdout_guard = Some(stdout_g); + stdout_layer_opt = Some( + tracing_subscriber::fmt::layer() + .with_timer(LocalTime::rfc_3339()) + .with_target(true) + .with_ansi(std::io::stdout().is_terminal()) + .with_thread_names(true) + .with_thread_ids(true) + .with_file(true) + .with_line_number(true) + .with_writer(stdout_nb) + .with_span_events(span_events) + .with_filter(build_env_filter(logger_level, None)), + ); + } + let filter = build_env_filter(logger_level, None); tracing_subscriber::registry() .with(filter) .with(ErrorLayer::default()) - .with(fmt_layer_opt) + .with(file_layer_opt) // File + .with(stdout_layer_opt) // Stdout (only if file logging enabled it) .with(tracer_layer) .with(otel_bridge) .with(metrics_layer) @@ -189,9 +299,9 @@ pub(super) fn init_observability_http( logger_provider, #[cfg(unix)] profiling_agent, - tracing_guard: None, - stdout_guard: None, - cleanup_handle: None, + tracing_guard, + stdout_guard, + cleanup_handle, }) } @@ -283,7 +393,7 @@ fn build_meter_provider( }) .build(); - global::set_meter_provider(provider.clone() as SdkMeterProvider); + global::set_meter_provider(provider.clone()); metrics::set_global_recorder(recorder).map_err(|e| TelemetryError::InstallMetricsRecorder(e.to_string()))?; OBSERVABILITY_METRIC_ENABLED.set(true).ok(); Ok(Some(provider)) diff --git a/crates/obs/src/telemetry/rolling.rs b/crates/obs/src/telemetry/rolling.rs index e7a57515..7d4b1749 100644 --- a/crates/obs/src/telemetry/rolling.rs +++ b/crates/obs/src/telemetry/rolling.rs @@ -136,24 +136,39 @@ impl RollingAppender { fs::create_dir_all(parent)?; } - // Open in append mode - let file = fs::OpenOptions::new().create(true).append(true).open(&path)?; + // Add retry logic to handle transient FS issues (e.g. anti-virus, indexing, or quick rename race) + const MAX_RETRIES: u32 = 3; + let mut last_error = None; - let meta = file.metadata()?; - self.size = meta.len(); + for i in 0..MAX_RETRIES { + // Open in append mode + match fs::OpenOptions::new().create(true).append(true).open(&path) { + Ok(file) => { + let meta = file.metadata()?; + self.size = meta.len(); - // Seed `last_roll_ts` from the file's modification time so that a - // process restart correctly triggers time-based rotation if the active - // file belongs to a previous period. - if let Ok(modified) = meta.modified() { - // Convert SystemTime to jiff::Timestamp - if let Ok(ts) = jiff::Timestamp::try_from(modified) { - self.last_roll_ts = ts.as_second(); + // Seed `last_roll_ts` from the file's modification time so that a + // process restart correctly triggers time-based rotation if the active + // file belongs to a previous period. + if let Ok(modified) = meta.modified() { + // Convert SystemTime to jiff::Timestamp + if let Ok(ts) = jiff::Timestamp::try_from(modified) { + self.last_roll_ts = ts.as_second(); + } + } + + self.file = Some(file); + return Ok(()); + } + Err(e) => { + last_error = Some(e); + // Exponential backoff: 10ms, 20ms, 40ms + thread::sleep(Duration::from_millis(10 * (1 << i))); + } } } - self.file = Some(file); - Ok(()) + Err(last_error.unwrap_or_else(|| io::Error::other("Failed to open log file after retries"))) } fn should_roll(&self, write_len: u64) -> bool { @@ -173,7 +188,12 @@ impl RollingAppender { fn roll(&mut self) -> io::Result<()> { // 1. Close current file first to ensure all buffers are flushed to OS (if any) // and handle released. - self.file = None; + if let Some(mut file) = self.file.take() + && let Err(e) = file.flush() + { + eprintln!("Failed to flush log file before rotation: {}", e); + return Err(e); + } let active_path = self.active_file_path(); if !active_path.exists() { @@ -222,10 +242,14 @@ impl RollingAppender { // Success! // 4. Reset state self.size = 0; - self.last_roll_ts = now.timestamp().as_second(); // 5. Re-open (creates new active file) self.open_file()?; + + // Explicitly update last_roll_ts to the rotation time. + // This overrides whatever open_file() derived from mtime, ensuring + // we stick to the logical rotation time. + self.last_roll_ts = now.timestamp().as_second(); return Ok(()); } Err(e) => { diff --git a/scripts/run.ps1 b/scripts/run.ps1 index 7d6275a4..c3f2a29b 100644 --- a/scripts/run.ps1 +++ b/scripts/run.ps1 @@ -15,7 +15,8 @@ $ErrorActionPreference = "Stop" # Check if ./rustfs/static/index.html exists -if (-not (Test-Path "./rustfs/static/index.html")) { +if (-not (Test-Path "./rustfs/static/index.html")) +{ Write-Host "Downloading rustfs-console-latest.zip" # Download rustfs-console-latest.zip Invoke-WebRequest -Uri "https://dl.rustfs.com/artifacts/console/rustfs-console-latest.zip" -OutFile "tempfile.zip" @@ -27,7 +28,8 @@ if (-not (Test-Path "./rustfs/static/index.html")) { Remove-Item "tempfile.zip" } -if (-not $env:SKIP_BUILD) { +if (-not $env:SKIP_BUILD) +{ cargo build -p rustfs --bins } @@ -39,7 +41,8 @@ Write-Host "Current directory: $current_dir" New-Item -ItemType Directory -Force -Path "./target/volume/test$_" | Out-Null } -if (-not $env:RUST_LOG) { +if (-not $env:RUST_LOG) +{ $env:RUST_BACKTRACE = "1" $env:RUST_LOG = "rustfs=debug,ecstore=info,s3s=debug,iam=info,notify=info" } @@ -76,9 +79,6 @@ $env:RUSTFS_OBS_LOG_STDOUT_ENABLED = "false" # Whether to enable local stdout lo $env:RUSTFS_OBS_LOG_DIRECTORY = "$current_dir/deploy/logs" # Log directory $env:RUSTFS_OBS_LOG_ROTATION_TIME = "hour" # Log rotation time unit $env:RUSTFS_OBS_LOG_ROTATION_SIZE_MB = "100" # Log rotation size in MB -$env:RUSTFS_OBS_LOG_POOL_CAPA = "10240" # Log pool capacity -$env:RUSTFS_OBS_LOG_MESSAGE_CAPA = "32768" # Log message capacity -$env:RUSTFS_OBS_LOG_FLUSH_MS = "300" # Log flush interval in milliseconds # tokio runtime $env:RUSTFS_RUNTIME_WORKER_THREADS = "16" @@ -180,12 +180,14 @@ $env:RUSTFS_FTPS_ENABLE = "false" # Reduce timeout for low-latency local storage $env:RUSTFS_LOCK_ACQUIRE_TIMEOUT = "30" -if ($args.Count -gt 0) { +if ($args.Count -gt 0) +{ $env:RUSTFS_VOLUMES = $args[0] } # Enable jemalloc for memory profiling -if (-not $env:MALLOC_CONF) { +if (-not $env:MALLOC_CONF) +{ $env:MALLOC_CONF = "prof:true,prof_active:true,lg_prof_sample:16,log:true,narenas:2,lg_chunk:21,background_thread:true,dirty_decay_ms:1000,muzzy_decay_ms:1000" } diff --git a/scripts/run.sh b/scripts/run.sh index e57e34dc..a0fdef7c 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -36,7 +36,7 @@ mkdir -p ./target/volume/test{1..4} if [ -z "$RUST_LOG" ]; then export RUST_BACKTRACE=1 - export RUST_LOG="rustfs=debug,ecstore=info,s3s=debug,iam=info,notify=info" + export RUST_LOG="info,rustfs=debug,rustfs_ecstore=info,s3s=debug,rustfs_iam=info,rustfs_notify=info" fi # export RUSTFS_ERASURE_SET_DRIVE_COUNT=5 @@ -69,18 +69,19 @@ export RUSTFS_CONSOLE_ADDRESS=":9001" #export RUSTFS_OBS_SERVICE_VERSION=0.1.0 # Service version export RUSTFS_OBS_ENVIRONMENT=production # Environment name development, staging, production export RUSTFS_OBS_LOGGER_LEVEL=info # Log level, supports trace, debug, info, warn, error -export RUSTFS_OBS_LOG_STDOUT_ENABLED=false # Whether to enable local stdout logging +export RUSTFS_OBS_LOG_STDOUT_ENABLED=true # Whether to enable local stdout logging export RUSTFS_OBS_LOG_DIRECTORY="$current_dir/deploy/logs" # Log directory export RUSTFS_OBS_LOG_ROTATION_TIME="minutely" # Log rotation time unit, can be "minutely", "hourly", "daily" -export RUSTFS_OBS_LOG_KEEP_FILES=30 # Number of log files to keep -export RUSTFS_OBS_LOG_MESSAGE_CAPA=32768 # Log message capacity -export RUSTFS_OBS_LOG_FLUSH_MS=300 # Log flush interval in milliseconds +export RUSTFS_OBS_LOG_KEEP_FILES=10 # Number of log files to keep +export RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=30 +export RUSTFS_OBS_LOG_MIN_FILE_AGE_SECONDS=60 +export RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS=7 + #tokio runtime export RUSTFS_RUNTIME_WORKER_THREADS=16 export RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024 export RUSTFS_RUNTIME_THREAD_PRINT_ENABLED=false -export RUSTFS_OBS_LOG_CLEANUP_INTERVAL_SECONDS=300 # shellcheck disable=SC2125 export RUSTFS_RUNTIME_THREAD_STACK_SIZE=1024*1024 export RUSTFS_RUNTIME_THREAD_KEEP_ALIVE=60