diff --git a/.docker/openobserve-otel/docker-compose.yml b/.docker/openobserve-otel/docker-compose.yml new file mode 100644 index 00000000..a4083bca --- /dev/null +++ b/.docker/openobserve-otel/docker-compose.yml @@ -0,0 +1,73 @@ +services: + openobserve: + image: public.ecr.aws/zinclabs/openobserve:latest + restart: unless-stopped + environment: + ZO_ROOT_USER_EMAIL: "root@rustfs.com" + ZO_ROOT_USER_PASSWORD: "rustfs123" + ZO_TRACING_HEADER_KEY: "Authorization" + ZO_TRACING_HEADER_VALUE: "Basic cm9vdEBydXN0ZnMuY29tOmQ4SXlCSEJTUkk3RGVlcEQ=" + ZO_DATA_DIR: "/data" + ZO_MEMORY_CACHE_ENABLED: "true" + ZO_MEMORY_CACHE_MAX_SIZE: "256" + RUST_LOG: "info" + TZ: Asia/Shanghai + ports: + - "5080:5080" + - "5081:5081" + volumes: + - ./data:/data + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:5080/health" ] + start_period: 60s + interval: 10s + timeout: 5s + retries: 6 + networks: + - otel-network + deploy: + resources: + limits: + memory: 1024M + reservations: + memory: 512M + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + restart: unless-stopped + environment: + - TZ=Asia/Shanghai + volumes: + - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + - "13133:13133" # Health check + - "1777:1777" # pprof + - "55679:55679" # zpages + - "1888:1888" # Metrics + - "8888:8888" # Prometheus metrics + - "8889:8889" # Additional metrics endpoint + depends_on: + - openobserve + networks: + - otel-network + deploy: + resources: + limits: + memory: 10240M + reservations: + memory: 512M + +networks: + otel-network: + driver: bridge + name: otel-network + ipam: + config: + - subnet: 172.28.0.0/16 + gateway: 172.28.0.1 + labels: + com.example.description: "Network for OpenObserve and OpenTelemetry Collector" +volumes: + data: \ No newline at end of file diff --git a/.docker/openobserve-otel/otel-collector-config.yaml b/.docker/openobserve-otel/otel-collector-config.yaml new file mode 100644 index 00000000..561692ad --- /dev/null +++ b/.docker/openobserve-otel/otel-collector-config.yaml @@ -0,0 +1,78 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + filelog: + include: [ "/var/log/app/*.log" ] + start_at: end + +processors: + batch: + timeout: 1s + send_batch_size: 1024 + memory_limiter: + check_interval: 1s + limit_mib: 400 + spike_limit_mib: 100 + +exporters: + otlphttp/openobserve: + endpoint: http://openobserve:5080/api/default # http://127.0.0.1:5080/api/default + headers: + Authorization: "Basic cm9vdEBydXN0ZnMuY29tOmQ4SXlCSEJTUkk3RGVlcEQ=" + stream-name: default + organization: default + compression: gzip + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + timeout: 10s + otlp/openobserve: + endpoint: openobserve:5081 # http://127.0.0.1:5080/api/default + headers: + Authorization: "Basic cm9vdEBydXN0ZnMuY29tOmQ4SXlCSEJTUkk3RGVlcEQ=" + stream-name: default + organization: default + compression: gzip + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + timeout: 10s + tls: + insecure: true + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + pprof: + endpoint: 0.0.0.0:1777 + zpages: + endpoint: 0.0.0.0:55679 + +service: + extensions: [ health_check, pprof, zpages ] + pipelines: + traces: + receivers: [ otlp ] + processors: [ memory_limiter, batch ] + exporters: [ otlp/openobserve ] + metrics: + receivers: [ otlp ] + processors: [ memory_limiter, batch ] + exporters: [ otlp/openobserve ] + logs: + receivers: [ otlp, filelog ] + processors: [ memory_limiter, batch ] + exporters: [ otlp/openobserve ] + telemetry: + logs: + level: "info" # Collector 日志级别 + metrics: + address: "0.0.0.0:8888" # Collector 自身指标暴露 diff --git a/.gitignore b/.gitignore index 46bfa8d4..d38ffae3 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ deploy/certs/* .env .rustfs.sys .cargo -profile.json \ No newline at end of file +profile.json +.docker/openobserve-otel/data \ No newline at end of file diff --git a/Cargo.lock b/Cargo.lock index 85ccd7d5..fdc90b69 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1745,6 +1745,15 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "crossbeam-queue" +version = "0.3.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f58bbc28f91df819d0aa2a2c00cd19754769c2fad90579b3592b1c9ba7a3115" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-utils" version = "0.8.21" @@ -3250,6 +3259,16 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +[[package]] +name = "erased-serde" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e004d887f51fcb9fef17317a2f3525c887d8aa3f4f50fed920816a688284a5b7" +dependencies = [ + "serde", + "typeid", +] + [[package]] name = "errno" version = "0.3.11" @@ -3364,6 +3383,27 @@ dependencies = [ "miniz_oxide", ] +[[package]] +name = "flexi_logger" +version = "0.30.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb03342077df16d5b1400d7bed00156882846d7a479ff61a6f10594bcc3423d8" +dependencies = [ + "chrono", + "crossbeam-channel", + "crossbeam-queue", + "log", + "notify-debouncer-mini", + "nu-ansi-term 0.50.1", + "regex", + "serde", + "serde_derive", + "thiserror 2.0.12", + "toml", + "tracing", + "tracing-subscriber", +] + [[package]] name = "flume" version = "0.11.1" @@ -3439,6 +3479,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "fsevent-sys" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2" +dependencies = [ + "libc", +] + [[package]] name = "futf" version = "0.1.5" @@ -4466,6 +4515,26 @@ dependencies = [ "cfb", ] +[[package]] +name = "inotify" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3" +dependencies = [ + "bitflags 2.9.0", + "inotify-sys", + "libc", +] + +[[package]] +name = "inotify-sys" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb" +dependencies = [ + "libc", +] + [[package]] name = "inout" version = "0.1.4" @@ -4682,6 +4751,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "kqueue" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a" +dependencies = [ + "kqueue-sys", + "libc", +] + +[[package]] +name = "kqueue-sys" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b" +dependencies = [ + "bitflags 1.3.2", + "libc", +] + [[package]] name = "kuchikiki" version = "0.8.2" @@ -4970,6 +5059,10 @@ name = "log" version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +dependencies = [ + "serde", + "value-bag", +] [[package]] name = "longest-increasing-subsequence" @@ -5181,6 +5274,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2886843bf800fba2e3377cff24abf6379b4c4d5c6681eaf9ea5b0d15090450bd" dependencies = [ "libc", + "log", "wasi 0.11.0+wasi-snapshot-preview1", "windows-sys 0.52.0", ] @@ -5350,6 +5444,43 @@ dependencies = [ "memchr", ] +[[package]] +name = "notify" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fee8403b3d66ac7b26aee6e40a897d85dc5ce26f44da36b8b73e987cc52e943" +dependencies = [ + "bitflags 2.9.0", + "filetime", + "fsevent-sys", + "inotify", + "kqueue", + "libc", + "log", + "mio", + "notify-types", + "walkdir", + "windows-sys 0.59.0", +] + +[[package]] +name = "notify-debouncer-mini" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a689eb4262184d9a1727f9087cd03883ea716682ab03ed24efec57d7716dccb8" +dependencies = [ + "log", + "notify", + "notify-types", + "tempfile", +] + +[[package]] +name = "notify-types" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d" + [[package]] name = "ntapi" version = "0.4.1" @@ -5369,6 +5500,15 @@ dependencies = [ "winapi", ] +[[package]] +name = "nu-ansi-term" +version = "0.50.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399" +dependencies = [ + "windows-sys 0.52.0", +] + [[package]] name = "nugine-rust-utils" version = "0.3.1" @@ -7464,8 +7604,8 @@ version = "0.0.1" dependencies = [ "async-trait", "chrono", - "config", - "local-ip-address", + "flexi_logger", + "nu-ansi-term 0.50.1", "nvml-wrapper", "opentelemetry", "opentelemetry-appender-tracing", @@ -7834,6 +7974,15 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "serde_fmt" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d4ddca14104cd60529e8c7f7ba71a2c8acd8f7f5cfcdc2faf97eeb7c3010a4" +dependencies = [ + "serde", +] + [[package]] name = "serde_json" version = "1.0.140" @@ -8395,6 +8544,84 @@ version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +[[package]] +name = "sval" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7cc9739f56c5d0c44a5ed45473ec868af02eb896af8c05f616673a31e1d1bb09" + +[[package]] +name = "sval_buffer" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f39b07436a8c271b34dad5070c634d1d3d76d6776e938ee97b4a66a5e8003d0b" +dependencies = [ + "sval", + "sval_ref", +] + +[[package]] +name = "sval_dynamic" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffcb072d857431bf885580dacecf05ed987bac931230736739a79051dbf3499b" +dependencies = [ + "sval", +] + +[[package]] +name = "sval_fmt" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f214f427ad94a553e5ca5514c95c6be84667cbc5568cce957f03f3477d03d5c" +dependencies = [ + "itoa 1.0.15", + "ryu", + "sval", +] + +[[package]] +name = "sval_json" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ed34b32e638dec9a99c8ac92d0aa1220d40041026b625474c2b6a4d6f4feb" +dependencies = [ + "itoa 1.0.15", + "ryu", + "sval", +] + +[[package]] +name = "sval_nested" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "14bae8fcb2f24fee2c42c1f19037707f7c9a29a0cda936d2188d48a961c4bb2a" +dependencies = [ + "sval", + "sval_buffer", + "sval_ref", +] + +[[package]] +name = "sval_ref" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a4eaea3821d3046dcba81d4b8489421da42961889902342691fb7eab491d79e" +dependencies = [ + "sval", +] + +[[package]] +name = "sval_serde" +version = "2.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "172dd4aa8cb3b45c8ac8f3b4111d644cd26938b0643ede8f93070812b87fb339" +dependencies = [ + "serde", + "sval", + "sval_nested", +] + [[package]] name = "syn" version = "1.0.109" @@ -9146,7 +9373,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" dependencies = [ "matchers", - "nu-ansi-term", + "nu-ansi-term 0.46.0", "once_cell", "regex", "serde", @@ -9242,6 +9469,12 @@ dependencies = [ "static_assertions", ] +[[package]] +name = "typeid" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc7d623258602320d5c55d1bc22793b57daff0ec7efc270ea7d55ce1d5f5471c" + [[package]] name = "typenum" version = "1.18.0" @@ -9410,6 +9643,42 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65" +[[package]] +name = "value-bag" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "943ce29a8a743eb10d6082545d861b24f9d1b160b7d741e0f2cdf726bec909c5" +dependencies = [ + "value-bag-serde1", + "value-bag-sval2", +] + +[[package]] +name = "value-bag-serde1" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35540706617d373b118d550d41f5dfe0b78a0c195dc13c6815e92e2638432306" +dependencies = [ + "erased-serde", + "serde", + "serde_fmt", +] + +[[package]] +name = "value-bag-sval2" +version = "1.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fe7e140a2658cc16f7ee7a86e413e803fc8f9b5127adc8755c19f9fefa63a52" +dependencies = [ + "sval", + "sval_buffer", + "sval_dynamic", + "sval_fmt", + "sval_json", + "sval_ref", + "sval_serde", +] + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index 31b95436..581e370b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,6 +75,7 @@ derive_builder = "0.20.2" dioxus = { version = "0.6.3", features = ["router"] } dirs = "6.0.0" flatbuffers = "25.2.10" +flexi_logger = { version = "0.30.2", features = ["trc"] } futures = "0.3.31" futures-core = "0.3.31" futures-util = "0.3.31" @@ -105,6 +106,7 @@ mime = "0.3.17" mime_guess = "2.0.5" netif = "0.1.6" nix = { version = "0.30.1", features = ["fs"] } +nu-ansi-term = "0.50.1" num_cpus = { version = "1.16.0" } nvml-wrapper = "0.10.0" object_store = "0.11.2" diff --git a/crates/config/src/constants/app.rs b/crates/config/src/constants/app.rs index c4f22f93..e6baaba8 100644 --- a/crates/config/src/constants/app.rs +++ b/crates/config/src/constants/app.rs @@ -56,14 +56,13 @@ pub const DEFAULT_ACCESS_KEY: &str = "rustfsadmin"; /// Example: RUSTFS_SECRET_KEY=rustfsadmin /// Example: --secret-key rustfsadmin pub const DEFAULT_SECRET_KEY: &str = "rustfsadmin"; -/// Default configuration file for observability -/// Default value: config/obs.toml -/// Environment variable: RUSTFS_OBS_CONFIG -/// Command line argument: --obs-config -/// Example: RUSTFS_OBS_CONFIG=config/obs.toml -/// Example: --obs-config config/obs.toml -/// Example: --obs-config /etc/rustfs/obs.toml -pub const DEFAULT_OBS_CONFIG: &str = "./deploy/config/obs.toml"; + +/// Default OBS configuration endpoint +/// Environment variable: DEFAULT_OBS_ENDPOINT +/// Command line argument: --obs-endpoint +/// Example: DEFAULT_OBS_ENDPOINT="http://localost:4317" +/// Example: --obs-endpoint http://localost:4317 +pub const DEFAULT_OBS_ENDPOINT: &str = ""; /// Default TLS key for rustfs /// This is the default key for TLS. @@ -90,6 +89,41 @@ pub const DEFAULT_CONSOLE_PORT: u16 = 9001; /// This is the default address for rustfs console. pub const DEFAULT_CONSOLE_ADDRESS: &str = concat!(":", DEFAULT_CONSOLE_PORT); +/// Default log filename for rustfs +/// This is the default log filename for rustfs. +/// It is used to store the logs of the application. +/// Default value: rustfs.log +/// Environment variable: RUSTFS_OBSERVABILITY_LOG_FILENAME +pub const DEFAULT_LOG_FILENAME: &str = "rustfs.log"; + +/// Default log directory for rustfs +/// This is the default log directory for rustfs. +/// It is used to store the logs of the application. +/// Default value: logs +/// Environment variable: RUSTFS_OBSERVABILITY_LOG_DIRECTORY +pub const DEFAULT_LOG_DIR: &str = "logs"; + +/// Default log rotation size mb for rustfs +/// This is the default log rotation size for rustfs. +/// It is used to rotate the logs of the application. +/// Default value: 100 MB +/// Environment variable: RUSTFS_OBSERVABILITY_LOG_ROTATION_SIZE_MB +pub const DEFAULT_LOG_ROTATION_SIZE_MB: u64 = 100; + +/// Default log rotation time for rustfs +/// This is the default log rotation time for rustfs. +/// It is used to rotate the logs of the application. +/// Default value: hour, eg: day,hour,minute,second +/// Environment variable: RUSTFS_OBSERVABILITY_LOG_ROTATION_TIME +pub const DEFAULT_LOG_ROTATION_TIME: &str = "day"; + +/// Default log keep files for rustfs +/// This is the default log keep files for rustfs. +/// It is used to keep the logs of the application. +/// Default value: 30 +/// Environment variable: RUSTFS_OBSERVABILITY_LOG_KEEP_FILES +pub const DEFAULT_LOG_KEEP_FILES: u16 = 30; + #[cfg(test)] mod tests { use super::*; @@ -154,10 +188,6 @@ mod tests { #[test] fn test_file_path_constants() { - // Test file path related constants - assert_eq!(DEFAULT_OBS_CONFIG, "./deploy/config/obs.toml"); - assert!(DEFAULT_OBS_CONFIG.ends_with(".toml"), "Config file should be TOML format"); - assert_eq!(RUSTFS_TLS_KEY, "rustfs_key.pem"); assert!(RUSTFS_TLS_KEY.ends_with(".pem"), "TLS key should be PEM format"); @@ -219,7 +249,6 @@ mod tests { ENVIRONMENT, DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, - DEFAULT_OBS_CONFIG, RUSTFS_TLS_KEY, RUSTFS_TLS_CERT, DEFAULT_ADDRESS, diff --git a/crates/obs/Cargo.toml b/crates/obs/Cargo.toml index 12c9831e..e2265103 100644 --- a/crates/obs/Cargo.toml +++ b/crates/obs/Cargo.toml @@ -20,8 +20,8 @@ kafka = ["dep:rdkafka"] rustfs-config = { workspace = true } async-trait = { workspace = true } chrono = { workspace = true } -config = { workspace = true } -local-ip-address = { workspace = true } +flexi_logger = { workspace = true, features = ["trc", "kv"] } +nu-ansi-term = { workspace = true } nvml-wrapper = { workspace = true, optional = true } opentelemetry = { workspace = true } opentelemetry-appender-tracing = { workspace = true, features = ["experimental_use_tracing_span_context", "experimental_metadata_attributes"] } diff --git a/crates/obs/src/config.rs b/crates/obs/src/config.rs index 770203de..b1f4850d 100644 --- a/crates/obs/src/config.rs +++ b/crates/obs/src/config.rs @@ -1,5 +1,7 @@ -use config::{Config, File, FileFormat}; -use rustfs_config::{APP_NAME, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT}; +use rustfs_config::{ + APP_NAME, DEFAULT_LOG_DIR, DEFAULT_LOG_FILENAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_SIZE_MB, + DEFAULT_LOG_ROTATION_TIME, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT, +}; use serde::{Deserialize, Serialize}; use std::env; @@ -22,54 +24,94 @@ pub struct OtelConfig { pub environment: Option, // Environment pub logger_level: Option, // Logger level pub local_logging_enabled: Option, // Local logging enabled -} - -/// Helper function: Extract observable configuration from environment variables -fn extract_otel_config_from_env() -> OtelConfig { - OtelConfig { - endpoint: env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()), - use_stdout: env::var("RUSTFS_OBSERVABILITY_USE_STDOUT") - .ok() - .and_then(|v| v.parse().ok()) - .or(Some(USE_STDOUT)), - sample_ratio: env::var("RUSTFS_OBSERVABILITY_SAMPLE_RATIO") - .ok() - .and_then(|v| v.parse().ok()) - .or(Some(SAMPLE_RATIO)), - meter_interval: env::var("RUSTFS_OBSERVABILITY_METER_INTERVAL") - .ok() - .and_then(|v| v.parse().ok()) - .or(Some(METER_INTERVAL)), - service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME") - .ok() - .and_then(|v| v.parse().ok()) - .or(Some(APP_NAME.to_string())), - service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION") - .ok() - .and_then(|v| v.parse().ok()) - .or(Some(SERVICE_VERSION.to_string())), - environment: env::var("RUSTFS_OBSERVABILITY_ENVIRONMENT") - .ok() - .and_then(|v| v.parse().ok()) - .or(Some(ENVIRONMENT.to_string())), - logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL") - .ok() - .and_then(|v| v.parse().ok()) - .or(Some(DEFAULT_LOG_LEVEL.to_string())), - local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED") - .ok() - .and_then(|v| v.parse().ok()) - .or(Some(false)), - } + // 新增 flexi_logger 相关配置 + pub log_directory: Option, // 日志文件目录 + pub log_filename: Option, // 日志文件名称 + pub log_rotation_size_mb: Option, // 日志文件大小切割阈值 (MB) + pub log_rotation_time: Option, // 日志按时间切割 (Hour, Day) + pub log_keep_files: Option, // 保留日志文件数量 } impl OtelConfig { + /// Helper function: Extract observable configuration from environment variables + pub fn extract_otel_config_from_env(endpoint: Option) -> OtelConfig { + let endpoint = if let Some(endpoint) = endpoint { + if endpoint.is_empty() { + env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()) + } else { + endpoint + } + } else { + env::var("RUSTFS_OBSERVABILITY_ENDPOINT").unwrap_or_else(|_| "".to_string()) + }; + let mut use_stdout = env::var("RUSTFS_OBSERVABILITY_USE_STDOUT") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(USE_STDOUT)); + if endpoint.is_empty() { + use_stdout = Some(true); + } + + OtelConfig { + endpoint, + use_stdout, + sample_ratio: env::var("RUSTFS_OBSERVABILITY_SAMPLE_RATIO") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(SAMPLE_RATIO)), + meter_interval: env::var("RUSTFS_OBSERVABILITY_METER_INTERVAL") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(METER_INTERVAL)), + service_name: env::var("RUSTFS_OBSERVABILITY_SERVICE_NAME") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(APP_NAME.to_string())), + service_version: env::var("RUSTFS_OBSERVABILITY_SERVICE_VERSION") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(SERVICE_VERSION.to_string())), + environment: env::var("RUSTFS_OBSERVABILITY_ENVIRONMENT") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(ENVIRONMENT.to_string())), + logger_level: env::var("RUSTFS_OBSERVABILITY_LOGGER_LEVEL") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(DEFAULT_LOG_LEVEL.to_string())), + local_logging_enabled: env::var("RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(false)), + log_directory: env::var("RUSTFS_OBSERVABILITY_LOG_DIRECTORY") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(DEFAULT_LOG_DIR.to_string())), + log_filename: env::var("RUSTFS_OBSERVABILITY_LOG_FILENAME") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(DEFAULT_LOG_FILENAME.to_string())), + log_rotation_size_mb: env::var("RUSTFS_OBSERVABILITY_LOG_ROTATION_SIZE_MB") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(DEFAULT_LOG_ROTATION_SIZE_MB)), // Default to 100 MB + log_rotation_time: env::var("RUSTFS_OBSERVABILITY_LOG_ROTATION_TIME") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(DEFAULT_LOG_ROTATION_TIME.to_string())), // Default to "Day" + log_keep_files: env::var("RUSTFS_OBSERVABILITY_LOG_KEEP_FILES") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(DEFAULT_LOG_KEEP_FILES)), // Default to keeping 30 log files + } + } + /// Create a new instance of OtelConfig with default values /// /// # Returns /// A new instance of OtelConfig pub fn new() -> Self { - extract_otel_config_from_env() + Self::extract_otel_config_from_env(None) } } @@ -122,6 +164,12 @@ pub struct WebhookSinkConfig { impl WebhookSinkConfig { pub fn new() -> Self { + Self::default() + } +} + +impl Default for WebhookSinkConfig { + fn default() -> Self { Self { endpoint: env::var("RUSTFS_SINKS_WEBHOOK_ENDPOINT") .ok() @@ -137,12 +185,6 @@ impl WebhookSinkConfig { } } -impl Default for WebhookSinkConfig { - fn default() -> Self { - Self::new() - } -} - /// File Sink Configuration - Add buffering parameters #[derive(Debug, Deserialize, Serialize, Clone)] pub struct FileSinkConfig { @@ -160,7 +202,6 @@ impl FileSinkConfig { eprintln!("Failed to create log directory: {}", e); return "rustfs/rustfs.log".to_string(); } - println!("Using log directory: {:?}", temp_dir); temp_dir .join("rustfs.log") .to_str() @@ -173,9 +214,18 @@ impl FileSinkConfig { .ok() .filter(|s| !s.trim().is_empty()) .unwrap_or_else(Self::get_default_log_path), - buffer_size: Some(8192), - flush_interval_ms: Some(1000), - flush_threshold: Some(100), + buffer_size: env::var("RUSTFS_SINKS_FILE_BUFFER_SIZE") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(8192)), + flush_interval_ms: env::var("RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(1000)), + flush_threshold: env::var("RUSTFS_SINKS_FILE_FLUSH_THRESHOLD") + .ok() + .and_then(|v| v.parse().ok()) + .or(Some(100)), } } } @@ -260,6 +310,14 @@ impl AppConfig { logger: Some(LoggerConfig::default()), } } + + pub fn new_with_endpoint(endpoint: Option) -> Self { + Self { + observability: OtelConfig::extract_otel_config_from_env(endpoint), + sinks: vec![SinkConfig::new()], + logger: Some(LoggerConfig::new()), + } + } } // implement default for AppConfig @@ -269,9 +327,6 @@ impl Default for AppConfig { } } -/// Default configuration file name -const DEFAULT_CONFIG_FILE: &str = "obs"; - /// Loading the configuration file /// Supports TOML, YAML and .env formats, read in order by priority /// @@ -288,51 +343,6 @@ const DEFAULT_CONFIG_FILE: &str = "obs"; /// /// let config = load_config(None); /// ``` -pub fn load_config(config_dir: Option) -> AppConfig { - let config_dir = if let Some(path) = config_dir { - // If a path is provided, check if it's empty - if path.is_empty() { - // If empty, use the default config file name - DEFAULT_CONFIG_FILE.to_string() - } else { - // Use the provided path - let path = std::path::Path::new(&path); - if path.extension().is_some() { - // If path has extension, use it as is (extension will be added by Config::builder) - path.with_extension("").to_string_lossy().into_owned() - } else { - // If path is a directory, append the default config file name - path.to_string_lossy().into_owned() - } - } - } else { - // If no path provided, use current directory + default config file - match env::current_dir() { - Ok(dir) => dir.join(DEFAULT_CONFIG_FILE).to_string_lossy().into_owned(), - Err(_) => { - eprintln!("Warning: Failed to get current directory, using default config file"); - DEFAULT_CONFIG_FILE.to_string() - } - } - }; - - // Log using proper logging instead of println when possible - println!("Using config file base: {}", config_dir); - - let app_config = Config::builder() - .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Toml).required(false)) - .add_source(File::with_name(config_dir.as_str()).format(FileFormat::Yaml).required(false)) - .build() - .unwrap_or_default(); - - match app_config.try_deserialize::() { - Ok(app_config) => { - println!("Parsed AppConfig: {:?}", app_config); - app_config - } - Err(e) => { - println!("Failed to deserialize config: {}", e); - AppConfig::default() - } - } +pub fn load_config(config: Option) -> AppConfig { + AppConfig::new_with_endpoint(config) } diff --git a/crates/obs/src/system/collector.rs b/crates/obs/src/system/collector.rs index c4184051..ea9bae3b 100644 --- a/crates/obs/src/system/collector.rs +++ b/crates/obs/src/system/collector.rs @@ -23,7 +23,7 @@ pub struct Collector { impl Collector { pub fn new(pid: Pid, meter: opentelemetry::metrics::Meter, interval_ms: u64) -> Result { - let mut system = System::new_all(); + let mut system = System::new(); let attributes = ProcessAttributes::new(pid, &mut system)?; let core_count = System::physical_core_count().ok_or(GlobalError::CoreCountError)?; let metrics = Metrics::new(&meter); @@ -52,7 +52,7 @@ impl Collector { fn collect(&mut self) -> Result<(), GlobalError> { self.system - .refresh_processes(sysinfo::ProcessesToUpdate::Some(&[self.pid]), true); + .refresh_processes(sysinfo::ProcessesToUpdate::Some(&[self.pid]), false); // refresh the network interface list and statistics self.networks.refresh(false); diff --git a/crates/obs/src/telemetry.rs b/crates/obs/src/telemetry.rs index d4ccaa82..ad021b74 100644 --- a/crates/obs/src/telemetry.rs +++ b/crates/obs/src/telemetry.rs @@ -1,4 +1,7 @@ +// Added flexi_logger related dependencies use crate::OtelConfig; +use flexi_logger::{style, Age, Cleanup, Criterion, DeferredNow, FileSpec, LogSpecification, Naming, Record, WriteMode}; +use nu_ansi_term::Color; use opentelemetry::trace::TracerProvider; use opentelemetry::{global, KeyValue}; use opentelemetry_appender_tracing::layer::OpenTelemetryTracingBridge; @@ -13,7 +16,9 @@ use opentelemetry_semantic_conventions::{ attribute::{DEPLOYMENT_ENVIRONMENT_NAME, NETWORK_LOCAL_ADDRESS, SERVICE_VERSION as OTEL_SERVICE_VERSION}, SCHEMA_URL, }; -use rustfs_config::{APP_NAME, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT}; +use rustfs_config::{ + APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, ENVIRONMENT, METER_INTERVAL, SAMPLE_RATIO, SERVICE_VERSION, USE_STDOUT, +}; use rustfs_utils::get_local_ip_with_default; use smallvec::SmallVec; use std::borrow::Cow; @@ -47,23 +52,44 @@ use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt, EnvFilte /// // When it's dropped, all telemetry components are properly shut down /// drop(otel_guard); /// ``` -#[derive(Debug)] +// Implement Debug trait correctly, rather than using derive, as some fields may not have implemented Debug pub struct OtelGuard { - tracer_provider: SdkTracerProvider, - meter_provider: SdkMeterProvider, - logger_provider: SdkLoggerProvider, + tracer_provider: Option, + meter_provider: Option, + logger_provider: Option, + // Add a flexi_logger handle to keep the logging alive + _flexi_logger_handles: Option, +} + +// Implement debug manually and avoid relying on all fields to implement debug +impl std::fmt::Debug for OtelGuard { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("OtelGuard") + .field("tracer_provider", &self.tracer_provider.is_some()) + .field("meter_provider", &self.meter_provider.is_some()) + .field("logger_provider", &self.logger_provider.is_some()) + .field("_flexi_logger_handles", &self._flexi_logger_handles.is_some()) + .finish() + } } impl Drop for OtelGuard { fn drop(&mut self) { - if let Err(err) = self.tracer_provider.shutdown() { - eprintln!("Tracer shutdown error: {:?}", err); + if let Some(provider) = self.tracer_provider.take() { + if let Err(err) = provider.shutdown() { + eprintln!("Tracer shutdown error: {:?}", err); + } } - if let Err(err) = self.meter_provider.shutdown() { - eprintln!("Meter shutdown error: {:?}", err); + + if let Some(provider) = self.meter_provider.take() { + if let Err(err) = provider.shutdown() { + eprintln!("Meter shutdown error: {:?}", err); + } } - if let Err(err) = self.logger_provider.shutdown() { - eprintln!("Logger shutdown error: {:?}", err); + if let Some(provider) = self.logger_provider.take() { + if let Err(err) = provider.shutdown() { + eprintln!("Logger shutdown error: {:?}", err); + } } } } @@ -106,156 +132,247 @@ pub fn init_telemetry(config: &OtelConfig) -> OtelGuard { let service_name = config.service_name.as_deref().unwrap_or(APP_NAME); let environment = config.environment.as_deref().unwrap_or(ENVIRONMENT); - // Pre-create resource objects to avoid repeated construction - let res = resource(config); + // Configure flexi_logger to cut by time and size + let mut flexi_logger_handle = None; + if !endpoint.is_empty() { + // Pre-create resource objects to avoid repeated construction + let res = resource(config); - // initialize tracer provider - let tracer_provider = { - let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO); - let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 { - Sampler::TraceIdRatioBased(sample_ratio) - } else { - Sampler::AlwaysOn + // initialize tracer provider + let tracer_provider = { + let sample_ratio = config.sample_ratio.unwrap_or(SAMPLE_RATIO); + let sampler = if sample_ratio > 0.0 && sample_ratio < 1.0 { + Sampler::TraceIdRatioBased(sample_ratio) + } else { + Sampler::AlwaysOn + }; + + let builder = SdkTracerProvider::builder() + .with_sampler(sampler) + .with_id_generator(RandomIdGenerator::default()) + .with_resource(res.clone()); + + let tracer_provider = if endpoint.is_empty() { + builder + .with_batch_exporter(opentelemetry_stdout::SpanExporter::default()) + .build() + } else { + let exporter = opentelemetry_otlp::SpanExporter::builder() + .with_tonic() + .with_endpoint(endpoint) + .build() + .unwrap(); + + let builder = if use_stdout { + builder + .with_batch_exporter(exporter) + .with_batch_exporter(opentelemetry_stdout::SpanExporter::default()) + } else { + builder.with_batch_exporter(exporter) + }; + + builder.build() + }; + + global::set_tracer_provider(tracer_provider.clone()); + tracer_provider }; - let builder = SdkTracerProvider::builder() - .with_sampler(sampler) - .with_id_generator(RandomIdGenerator::default()) - .with_resource(res.clone()); + // initialize meter provider + let meter_provider = { + let mut builder = MeterProviderBuilder::default().with_resource(res.clone()); - let tracer_provider = if endpoint.is_empty() { - builder - .with_batch_exporter(opentelemetry_stdout::SpanExporter::default()) - .build() - } else { - let exporter = opentelemetry_otlp::SpanExporter::builder() - .with_tonic() - .with_endpoint(endpoint) - .build() - .unwrap(); - - let builder = if use_stdout { - builder - .with_batch_exporter(exporter) - .with_batch_exporter(opentelemetry_stdout::SpanExporter::default()) + if endpoint.is_empty() { + builder = builder.with_reader(create_periodic_reader(meter_interval)); } else { - builder.with_batch_exporter(exporter) - }; + let exporter = opentelemetry_otlp::MetricExporter::builder() + .with_tonic() + .with_endpoint(endpoint) + .with_temporality(opentelemetry_sdk::metrics::Temporality::default()) + .build() + .unwrap(); + + builder = builder.with_reader( + PeriodicReader::builder(exporter) + .with_interval(std::time::Duration::from_secs(meter_interval)) + .build(), + ); + + if use_stdout { + builder = builder.with_reader(create_periodic_reader(meter_interval)); + } + } + + let meter_provider = builder.build(); + global::set_meter_provider(meter_provider.clone()); + meter_provider + }; + + // initialize logger provider + let logger_provider = { + let mut builder = SdkLoggerProvider::builder().with_resource(res); + + if endpoint.is_empty() { + builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default()); + } else { + let exporter = opentelemetry_otlp::LogExporter::builder() + .with_tonic() + .with_endpoint(endpoint) + .build() + .unwrap(); + + builder = builder.with_batch_exporter(exporter); + + if use_stdout { + builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default()); + } + } builder.build() }; - global::set_tracer_provider(tracer_provider.clone()); - tracer_provider - }; + // configuring tracing + { + // configure the formatting layer + let fmt_layer = { + let enable_color = std::io::stdout().is_terminal(); + let mut layer = tracing_subscriber::fmt::layer() + .with_target(true) + .with_ansi(enable_color) + .with_thread_names(true) + .with_thread_ids(true) + .with_file(true) + .with_line_number(true); - // initialize meter provider - let meter_provider = { - let mut builder = MeterProviderBuilder::default().with_resource(res.clone()); + // Only add full span events tracking in the development environment + if environment != ENVIRONMENT { + layer = layer.with_span_events(FmtSpan::FULL); + } - if endpoint.is_empty() { - builder = builder.with_reader(create_periodic_reader(meter_interval)); - } else { - let exporter = opentelemetry_otlp::MetricExporter::builder() - .with_tonic() - .with_endpoint(endpoint) - .with_temporality(opentelemetry_sdk::metrics::Temporality::default()) - .build() - .unwrap(); + layer.with_filter(build_env_filter(logger_level, None)) + }; - builder = builder.with_reader( - PeriodicReader::builder(exporter) - .with_interval(std::time::Duration::from_secs(meter_interval)) - .build(), - ); + let filter = build_env_filter(logger_level, None); + let otel_filter = build_env_filter(logger_level, None); + let otel_layer = OpenTelemetryTracingBridge::new(&logger_provider).with_filter(otel_filter); + let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string()); - if use_stdout { - builder = builder.with_reader(create_periodic_reader(meter_interval)); + // Configure registry to avoid repeated calls to filter methods + tracing_subscriber::registry() + .with(filter) + .with(ErrorLayer::default()) + .with(if config.local_logging_enabled.unwrap_or(false) { + Some(fmt_layer) + } else { + None + }) + .with(OpenTelemetryLayer::new(tracer)) + .with(otel_layer) + .with(MetricsLayer::new(meter_provider.clone())) + .init(); + + if !endpoint.is_empty() { + info!( + "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {},RUST_LOG env: {}", + endpoint, + logger_level, + std::env::var("RUST_LOG").unwrap_or_else(|_| "Not set".to_string()) + ); } } - let meter_provider = builder.build(); - global::set_meter_provider(meter_provider.clone()); - meter_provider - }; - - // initialize logger provider - let logger_provider = { - let mut builder = SdkLoggerProvider::builder().with_resource(res); - - if endpoint.is_empty() { - builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default()); - } else { - let exporter = opentelemetry_otlp::LogExporter::builder() - .with_tonic() - .with_endpoint(endpoint) - .build() - .unwrap(); - - builder = builder.with_batch_exporter(exporter); - - if use_stdout { - builder = builder.with_batch_exporter(opentelemetry_stdout::LogExporter::default()); - } + OtelGuard { + tracer_provider: Some(tracer_provider), + meter_provider: Some(meter_provider), + logger_provider: Some(logger_provider), + _flexi_logger_handles: flexi_logger_handle, } + } else { + // Obtain the log directory and file name configuration + let log_directory = config.log_directory.as_deref().unwrap_or("logs"); + let log_filename = config.log_filename.as_deref().unwrap_or(service_name); - builder.build() - }; - - // configuring tracing - { - // configure the formatting layer - let fmt_layer = { - let enable_color = std::io::stdout().is_terminal(); - let mut layer = tracing_subscriber::fmt::layer() - .with_target(true) - .with_ansi(enable_color) - .with_thread_names(true) - .with_thread_ids(true) - .with_file(true) - .with_line_number(true); - - // Only add full span events tracking in the development environment - if environment != ENVIRONMENT { - layer = layer.with_span_events(FmtSpan::FULL); + // Build log cutting conditions + let rotation_criterion = match (config.log_rotation_time.as_deref(), config.log_rotation_size_mb) { + // Cut by time and size at the same time + (Some(time), Some(size)) => { + let age = match time.to_lowercase().as_str() { + "hour" => Age::Hour, + "day" => Age::Day, + "minute" => Age::Minute, + "second" => Age::Second, + _ => Age::Day, // The default is by day + }; + Criterion::AgeOrSize(age, size * 1024 * 1024) // Convert to bytes } - - layer.with_filter(build_env_filter(logger_level, None)) + // Cut by time only + (Some(time), None) => { + let age = match time.to_lowercase().as_str() { + "hour" => Age::Hour, + "day" => Age::Day, + "minute" => Age::Minute, + "second" => Age::Second, + _ => Age::Day, // The default is by day + }; + Criterion::Age(age) + } + // Cut by size only + (None, Some(size)) => { + Criterion::Size(size * 1024 * 1024) // Convert to bytes + } + // By default, it is cut by the day + _ => Criterion::Age(Age::Day), }; - let filter = build_env_filter(logger_level, None); - let otel_filter = build_env_filter(logger_level, None); - let otel_layer = OpenTelemetryTracingBridge::new(&logger_provider).with_filter(otel_filter); - let tracer = tracer_provider.tracer(Cow::Borrowed(service_name).to_string()); + // The number of log files retained + let keep_files = config.log_keep_files.unwrap_or(DEFAULT_LOG_KEEP_FILES); - // Configure registry to avoid repeated calls to filter methods - tracing_subscriber::registry() - .with(filter) - .with(ErrorLayer::default()) - .with(if config.local_logging_enabled.unwrap_or(false) { - Some(fmt_layer) - } else { - None - }) - .with(OpenTelemetryLayer::new(tracer)) - .with(otel_layer) - .with(MetricsLayer::new(meter_provider.clone())) - .init(); + // Parsing the log level + let log_spec = LogSpecification::parse(logger_level).unwrap_or(LogSpecification::info()); - if !endpoint.is_empty() { - info!( - "OpenTelemetry telemetry initialized with OTLP endpoint: {}, logger_level: {},RUST_LOG env: {}", - endpoint, - logger_level, - std::env::var("RUST_LOG").unwrap_or_else(|_| "Not set".to_string()) - ); + // Configure the flexi_logger + let flexi_logger_result = flexi_logger::Logger::with(log_spec) + .log_to_file( + FileSpec::default() + .directory(log_directory) + .basename(log_filename) + .suffix("log"), + ) + .rotate(rotation_criterion, Naming::Timestamps, Cleanup::KeepLogFiles(keep_files.into())) + .format_for_files(format_for_file) // Add a custom formatting function for file output + .duplicate_to_stdout(flexi_logger::Duplicate::Info) + .format_for_stdout(format_with_color) // Add a custom formatting function for terminal output + .write_mode(WriteMode::Async) + .start(); + + if let Ok(logger) = flexi_logger_result { + // Save the logger handle to keep the logging + flexi_logger_handle = Some(logger); + + info!("Flexi logger initialized with file logging to {}/{}.log", log_directory, log_filename); + + // Log logging of log cutting conditions + match (config.log_rotation_time.as_deref(), config.log_rotation_size_mb) { + (Some(time), Some(size)) => info!( + "Log rotation configured for: every {} or when size exceeds {}MB, keeping {} files", + time, size, keep_files + ), + (Some(time), None) => info!("Log rotation configured for: every {}, keeping {} files", time, keep_files), + (None, Some(size)) => { + info!("Log rotation configured for: when size exceeds {}MB, keeping {} files", size, keep_files) + } + _ => info!("Log rotation configured for: daily, keeping {} files", keep_files), + } + } else { + eprintln!("Failed to initialize flexi_logger: {:?}", flexi_logger_result.err()); } - } - OtelGuard { - tracer_provider, - meter_provider, - logger_provider, + OtelGuard { + tracer_provider: None, + meter_provider: None, + logger_provider: None, + _flexi_logger_handles: flexi_logger_handle, + } } } @@ -271,3 +388,50 @@ fn build_env_filter(logger_level: &str, default_level: Option<&str>) -> EnvFilte filter } + +/// Custom Log Formatter Function - Terminal Output (with Color) +fn format_with_color(w: &mut dyn std::io::Write, now: &mut DeferredNow, record: &Record) -> Result<(), std::io::Error> { + let level = record.level(); + let level_style = style(level); + + // Get the current thread information + let binding = std::thread::current(); + let thread_name = binding.name().unwrap_or("unnamed"); + let thread_id = format!("{:?}", std::thread::current().id()); + + writeln!( + w, + "{} {} [{}] [{}:{}] [{}:{}] {}", + now.now().format("%Y-%m-%d %H:%M:%S%.6f"), + level_style.paint(level.to_string()), + Color::Magenta.paint(record.target()), + Color::Blue.paint(record.file().unwrap_or("unknown")), + Color::Blue.paint(record.line().unwrap_or(0).to_string()), + Color::Green.paint(thread_name), + Color::Green.paint(thread_id), + record.args() + ) +} + +/// Custom Log Formatter - File Output (No Color) +fn format_for_file(w: &mut dyn std::io::Write, now: &mut DeferredNow, record: &Record) -> Result<(), std::io::Error> { + let level = record.level(); + + // Get the current thread information + let binding = std::thread::current(); + let thread_name = binding.name().unwrap_or("unnamed"); + let thread_id = format!("{:?}", std::thread::current().id()); + + writeln!( + w, + "{} {} [{}] [{}:{}] [{}:{}] {}", + now.now().format("%Y-%m-%d %H:%M:%S%.6f"), + level, + record.target(), + record.file().unwrap_or("unknown"), + record.line().unwrap_or(0), + thread_name, + thread_id, + record.args() + ) +} diff --git a/rustfs/src/config/mod.rs b/rustfs/src/config/mod.rs index a93b6dda..a720221f 100644 --- a/rustfs/src/config/mod.rs +++ b/rustfs/src/config/mod.rs @@ -64,8 +64,8 @@ pub struct Opt { /// Observability configuration file /// Default value: config/obs.toml - #[arg(long, default_value_t = rustfs_config::DEFAULT_OBS_CONFIG.to_string(), env = "RUSTFS_OBS_CONFIG")] - pub obs_config: String, + #[arg(long, default_value_t = rustfs_config::DEFAULT_OBS_ENDPOINT.to_string(), env = "RUSTFS_OBS_ENDPOINT")] + pub obs_endpoint: String, /// tls path for rustfs api and console. #[arg(long, env = "RUSTFS_TLS_PATH")] diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index dca93c24..b3192be3 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -103,7 +103,7 @@ async fn main() -> Result<()> { init_license(opt.license.clone()); // Load the configuration file - let config = load_config(Some(opt.clone().obs_config)); + let config = load_config(Some(opt.clone().obs_endpoint)); // Initialize Observability let (_logger, guard) = init_obs(config.clone()).await; diff --git a/scripts/run.sh b/scripts/run.sh index 2004e05a..aa8258b5 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -31,44 +31,45 @@ export RUSTFS_VOLUMES="./target/volume/test{0...4}" # export RUSTFS_VOLUMES="./target/volume/test" export RUSTFS_ADDRESS=":9000" export RUSTFS_CONSOLE_ENABLE=true -export RUSTFS_CONSOLE_ADDRESS=":9002" +export RUSTFS_CONSOLE_ADDRESS=":9001" # export RUSTFS_SERVER_DOMAINS="localhost:9000" # HTTPS 证书目录 # export RUSTFS_TLS_PATH="./deploy/certs" # 具体路径修改为配置文件真实路径，obs.example.toml 仅供参考 其中 `RUSTFS_OBS_CONFIG` 和下面变量二选一 -export RUSTFS_OBS_CONFIG="./deploy/config/obs.example.toml" +# export RUSTFS_OBS_ENDPOINT=http://localhost:4317 # 如下变量需要必须参数都有值才可以，以及会覆盖配置文件中的值 -#export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 -#export RUSTFS__OBSERVABILITY__USE_STDOUT=false -#export RUSTFS__OBSERVABILITY__SAMPLE_RATIO=2.0 -#export RUSTFS__OBSERVABILITY__METER_INTERVAL=31 -#export RUSTFS__OBSERVABILITY__SERVICE_NAME=rustfs -#export RUSTFS__OBSERVABILITY__SERVICE_VERSION=0.1.0 -#export RUSTFS__OBSERVABILITY__ENVIRONMENT=develop -#export RUSTFS__OBSERVABILITY__LOGGER_LEVEL=debug -#export RUSTFS__OBSERVABILITY__LOCAL_LOGGING_ENABLED=true +#export RUSTFS_OBSERVABILITY_ENDPOINT=http://localhost:4317 # OpenTelemetry Collector 的地址 +#export RUSTFS_OBSERVABILITY_USE_STDOUT=false # 是否使用标准输出 +#export RUSTFS_OBSERVABILITY_SAMPLE_RATIO=2.0 # 采样率，0.0-1.0之间，0.0表示不采样，1.0表示全部采样 +#export RUSTFS_OBSERVABILITY_METER_INTERVAL=1 # 采样间隔，单位为秒 +#export RUSTFS_OBSERVABILITY_SERVICE_NAME=rustfs # 服务名称 +#export RUSTFS_OBSERVABILITY_SERVICE_VERSION=0.1.0 # 服务版本 +#export RUSTFS_OBSERVABILITY_ENVIRONMENT=develop # 环境名称 +#export RUSTFS_OBSERVABILITY_LOGGER_LEVEL=debug # 日志级别，支持 trace, debug, info, warn, error +export RUSTFS_OBSERVABILITY_LOCAL_LOGGING_ENABLED=true # 是否启用本地日志记录 +export RUSTFS_OBSERVABILITY_LOG_DIRECTORY="./deploy/logs" # Log directory +export RUSTFS_OBSERVABILITY_LOG_ROTATION_TIME="minute" # Log rotation time unit, can be "second", "minute", "hour", "day" +export RUSTFS_OBSERVABILITY_LOG_ROTATION_SIZE_MB=1 # Log rotation size in MB + # -#export RUSTFS__SINKS_0__type=File -#export RUSTFS__SINKS_0__path=./deploy/logs/rustfs.log -#export RUSTFS__SINKS_0__buffer_size=12 -#export RUSTFS__SINKS_0__flush_interval_ms=1000 -#export RUSTFS__SINKS_0__flush_threshold=100 +#export RUSTFS_SINKS_FILE_PATH=./deploy/logs/rustfs.log +#export RUSTFS_SINKS_FILE_BUFFER_SIZE=12 +#export RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS=1000 +#export RUSTFS_SINKS_FILE_FLUSH_THRESHOLD=100 # -#export RUSTFS__SINKS_1__type=Kakfa -#export RUSTFS__SINKS_1__brokers=localhost:9092 -#export RUSTFS__SINKS_1__topic=logs -#export RUSTFS__SINKS_1__batch_size=100 -#export RUSTFS__SINKS_1__batch_timeout_ms=1000 +#export RUSTFS_SINKS_KAFKA_BROKERS=localhost:9092 +#export RUSTFS_SINKS_KAFKA_TOPIC=logs +#export RUSTFS_SINKS_KAFKA_BATCH_SIZE=100 +#export RUSTFS_SINKS_KAFKA_BATCH_TIMEOUT_MS=1000 # -#export RUSTFS__SINKS_2__type=Webhook -#export RUSTFS__SINKS_2__endpoint=http://localhost:8080/webhook -#export RUSTFS__SINKS_2__auth_token=you-auth-token -#export RUSTFS__SINKS_2__batch_size=100 -#export RUSTFS__SINKS_2__batch_timeout_ms=1000 +#export RUSTFS_SINKS_WEBHOOK_ENDPOINT=http://localhost:8080/webhook +#export RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN=you-auth-token +#export RUSTFS_SINKS_WEBHOOK_BATCH_SIZE=100 +#export RUSTFS_SINKS_WEBHOOK_BATCH_TIMEOUT_MS=1000 # -#export RUSTFS__LOGGER__QUEUE_CAPACITY=10 +#export RUSTFS_LOGGER_QUEUE_CAPACITY=10 export OTEL_INSTRUMENTATION_NAME="rustfs" export OTEL_INSTRUMENTATION_VERSION="0.1.1"