From 8f227b2691f1f1d56b52ed747a9a009c989e5e8a Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 17 Dec 2025 11:32:54 +0800 Subject: [PATCH] fix logger --- crates/config/src/constants/mod.rs | 1 + crates/config/src/constants/scanner.rs | 28 +++++++++++++++++++ crates/config/src/lib.rs | 2 ++ crates/ecstore/src/disk/error_conv.rs | 1 - crates/ecstore/src/disk/local.rs | 2 +- crates/scanner/src/scanner.rs | 11 ++++++-- scripts/run.sh | 38 +++++++++++++------------- 7 files changed, 59 insertions(+), 24 deletions(-) create mode 100644 crates/config/src/constants/scanner.rs diff --git a/crates/config/src/constants/mod.rs b/crates/config/src/constants/mod.rs index 7f6dbff9..a03e03ad 100644 --- a/crates/config/src/constants/mod.rs +++ b/crates/config/src/constants/mod.rs @@ -21,5 +21,6 @@ pub(crate) mod heal; pub(crate) mod object; pub(crate) mod profiler; pub(crate) mod runtime; +pub(crate) mod scanner; pub(crate) mod targets; pub(crate) mod tls; diff --git a/crates/config/src/constants/scanner.rs b/crates/config/src/constants/scanner.rs new file mode 100644 index 00000000..ff024150 --- /dev/null +++ b/crates/config/src/constants/scanner.rs @@ -0,0 +1,28 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/// Environment variable name that specifies the data scanner start delay in seconds. +/// - Purpose: Define the delay between data scanner operations. +/// - Unit: seconds (u64). +/// - Valid values: any positive integer. +/// - Semantics: This delay controls how frequently the data scanner checks for and processes data; shorter delays lead to more responsive scanning but may increase system load. +/// - Example: `export RUSTFS_DATA_SCANNER_START_DELAY_SECS=10` +/// - Note: Choose an appropriate delay that balances scanning responsiveness with overall system performance. +pub const ENV_DATA_SCANNER_START_DELAY_SECS: &str = "RUSTFS_DATA_SCANNER_START_DELAY_SECS"; + +/// Default data scanner start delay in seconds if not specified in the environment variable. +/// - Value: 10 seconds. +/// - Rationale: This default interval provides a reasonable balance between scanning responsiveness and system load for most deployments. +/// - Adjustments: Users may modify this value via the `RUSTFS_DATA_SCANNER_START_DELAY_SECS` environment variable based on their specific scanning requirements and system performance. +pub const DEFAULT_DATA_SCANNER_START_DELAY_SECS: u64 = 60; diff --git a/crates/config/src/lib.rs b/crates/config/src/lib.rs index 9d83800e..d2278fef 100644 --- a/crates/config/src/lib.rs +++ b/crates/config/src/lib.rs @@ -33,6 +33,8 @@ pub use constants::profiler::*; #[cfg(feature = "constants")] pub use constants::runtime::*; #[cfg(feature = "constants")] +pub use constants::scanner::*; +#[cfg(feature = "constants")] pub use constants::targets::*; #[cfg(feature = "constants")] pub use constants::tls::*; diff --git a/crates/ecstore/src/disk/error_conv.rs b/crates/ecstore/src/disk/error_conv.rs index 2503fae1..ed8487a9 100644 --- a/crates/ecstore/src/disk/error_conv.rs +++ b/crates/ecstore/src/disk/error_conv.rs @@ -15,7 +15,6 @@ use super::error::DiskError; pub fn to_file_error(io_err: std::io::Error) -> std::io::Error { - tracing::warn!("to_file_error: io_err: {:?}", io_err); match io_err.kind() { std::io::ErrorKind::NotFound => DiskError::FileNotFound.into(), std::io::ErrorKind::PermissionDenied => DiskError::FileAccessDenied.into(), diff --git a/crates/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs index 2ffe2671..a1283ec7 100644 --- a/crates/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -683,7 +683,7 @@ impl LocalDisk { return Err(DiskError::FileNotFound); } - warn!("read_raw: file_path: {:?}", file_path.as_ref()); + debug!("read_raw: file_path: {:?}", file_path.as_ref()); let meta_path = file_path.as_ref().join(Path::new(STORAGE_FORMAT_FILE)); diff --git a/crates/scanner/src/scanner.rs b/crates/scanner/src/scanner.rs index 637a1fca..f90538e1 100644 --- a/crates/scanner/src/scanner.rs +++ b/crates/scanner/src/scanner.rs @@ -21,6 +21,7 @@ use crate::scanner_io::ScannerIO; use crate::{DataUsageInfo, ScannerError}; use chrono::{DateTime, Utc}; use rustfs_common::heal_channel::HealScanMode; +use rustfs_config::{DEFAULT_DATA_SCANNER_START_DELAY_SECS, ENV_DATA_SCANNER_START_DELAY_SECS}; use rustfs_ecstore::config::com::{read_config, save_config}; use rustfs_ecstore::error::Error as EcstoreError; use rustfs_ecstore::global::is_erasure_sd; @@ -32,7 +33,11 @@ use tokio_util::sync::CancellationToken; use tracing::{debug, error, info, warn}; const DATA_USAGE_UPDATE_DIR_CYCLES: usize = 16; -const DATA_SCANNER_START_DELAY: Duration = Duration::from_secs(10); + +fn data_scanner_start_delay() -> Duration { + let secs = rustfs_utils::get_env_u64(ENV_DATA_SCANNER_START_DELAY_SECS, DEFAULT_DATA_SCANNER_START_DELAY_SECS); + Duration::from_secs(secs) +} pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc) { let ctx_clone = ctx.clone(); @@ -46,7 +51,7 @@ pub async fn init_data_scanner(ctx: CancellationToken, storeapi: Arc) { if let Err(e) = run_data_scanner(ctx_clone.clone(), storeapi_clone.clone()).await { error!("Failed to run data scanner: {e}"); } - tokio::time::sleep(DATA_SCANNER_START_DELAY).await; + tokio::time::sleep(data_scanner_start_delay()).await; } }); } @@ -132,7 +137,7 @@ pub async fn run_data_scanner(ctx: CancellationToken, storeapi: Arc) -> } } - let mut ticker = tokio::time::interval(DATA_SCANNER_START_DELAY); + let mut ticker = tokio::time::interval(data_scanner_start_delay()); loop { tokio::select! { _ = ctx.cancelled() => { diff --git a/scripts/run.sh b/scripts/run.sh index 6ac6039c..46b335d6 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -88,30 +88,30 @@ export OTEL_INSTRUMENTATION_VERSION="0.1.1" export OTEL_INSTRUMENTATION_SCHEMA_URL="https://opentelemetry.io/schemas/1.31.0" export OTEL_INSTRUMENTATION_ATTRIBUTES="env=production" -# notify -export RUSTFS_NOTIFY_WEBHOOK_ENABLE="on" # Whether to enable webhook notification -export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT="http://[::]:3020/webhook" # Webhook notification address -export RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR="$current_dir/deploy/logs/notify" +# # notify +# export RUSTFS_NOTIFY_WEBHOOK_ENABLE="on" # Whether to enable webhook notification +# export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT="http://[::]:3020/webhook" # Webhook notification address +# export RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR="$current_dir/deploy/logs/notify" -export RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY="on" # Whether to enable webhook notification -export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY="http://[::]:3020/webhook" # Webhook notification address -export RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_PRIMARY="$current_dir/deploy/logs/notify" +# export RUSTFS_NOTIFY_WEBHOOK_ENABLE_PRIMARY="on" # Whether to enable webhook notification +# export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_PRIMARY="http://[::]:3020/webhook" # Webhook notification address +# export RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_PRIMARY="$current_dir/deploy/logs/notify" -export RUSTFS_NOTIFY_WEBHOOK_ENABLE_MASTER="on" # Whether to enable webhook notification -export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MASTER="http://[::]:3020/webhook" # Webhook notification address -export RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_MASTER="$current_dir/deploy/logs/notify" +# export RUSTFS_NOTIFY_WEBHOOK_ENABLE_MASTER="on" # Whether to enable webhook notification +# export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MASTER="http://[::]:3020/webhook" # Webhook notification address +# export RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_MASTER="$current_dir/deploy/logs/notify" -export RUSTFS_AUDIT_WEBHOOK_ENABLE="on" # Whether to enable webhook audit -export RUSTFS_AUDIT_WEBHOOK_ENDPOINT="http://[::]:3020/webhook" # Webhook audit address -export RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR="$current_dir/deploy/logs/audit" +# export RUSTFS_AUDIT_WEBHOOK_ENABLE="on" # Whether to enable webhook audit +# export RUSTFS_AUDIT_WEBHOOK_ENDPOINT="http://[::]:3020/webhook" # Webhook audit address +# export RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR="$current_dir/deploy/logs/audit" -export RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARY="on" # Whether to enable webhook audit -export RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY="http://[::]:3020/webhook" # Webhook audit address -export RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_PRIMARY="$current_dir/deploy/logs/audit" +# export RUSTFS_AUDIT_WEBHOOK_ENABLE_PRIMARY="on" # Whether to enable webhook audit +# export RUSTFS_AUDIT_WEBHOOK_ENDPOINT_PRIMARY="http://[::]:3020/webhook" # Webhook audit address +# export RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_PRIMARY="$current_dir/deploy/logs/audit" -export RUSTFS_AUDIT_WEBHOOK_ENABLE_MASTER="on" # Whether to enable webhook audit -export RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MASTER="http://[::]:3020/webhook" # Webhook audit address -export RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_MASTER="$current_dir/deploy/logs/audit" +# export RUSTFS_AUDIT_WEBHOOK_ENABLE_MASTER="on" # Whether to enable webhook audit +# export RUSTFS_AUDIT_WEBHOOK_ENDPOINT_MASTER="http://[::]:3020/webhook" # Webhook audit address +# export RUSTFS_AUDIT_WEBHOOK_QUEUE_DIR_MASTER="$current_dir/deploy/logs/audit" # export RUSTFS_POLICY_PLUGIN_URL="http://localhost:8181/v1/data/rustfs/authz/allow" # The URL of the OPA system # export RUSTFS_POLICY_PLUGIN_AUTH_TOKEN="your-opa-token" # The authentication token for the OPA system is optional