From fe15655f1e2ef52ea6455f0f57c1151cd5d54bb9 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 29 May 2025 08:34:17 +0800 Subject: [PATCH 1/2] init openobserve docker yml --- .docker/openobserve-otel/docker-compose.yml | 66 +++++++++++++++++++ .../otel-collector-config.yaml | 58 ++++++++++++++++ scripts/run.sh | 2 +- 3 files changed, 125 insertions(+), 1 deletion(-) create mode 100644 .docker/openobserve-otel/docker-compose.yml create mode 100644 .docker/openobserve-otel/otel-collector-config.yaml diff --git a/.docker/openobserve-otel/docker-compose.yml b/.docker/openobserve-otel/docker-compose.yml new file mode 100644 index 00000000..13d77625 --- /dev/null +++ b/.docker/openobserve-otel/docker-compose.yml @@ -0,0 +1,66 @@ +services: + openobserve: + image: public.ecr.aws/zinclabs/openobserve:latest + restart: unless-stopped + environment: + ZO_ROOT_USER_EMAIL: "root@rustfs.com" + ZO_ROOT_USER_PASSWORD: "rustfs123" + ZO_TRACING_HEADER_KEY: "Authorization" + ZO_TRACING_HEADER_VALUE: "Bearer cm9vdEBydXN0ZnMuY29tOmxIV0RqQmZMWXJ6MnZOajU=" + ZO_DATA_DIR: "/data" + ZO_MEMORY_CACHE_ENABLED: "true" + ZO_MEMORY_CACHE_MAX_SIZE: "256" + RUST_LOG: "info" + ports: + - "5080:5080" + volumes: + - ./data:/data + healthcheck: + test: [ "CMD", "curl", "-f", "http://localhost:5080/health" ] + interval: 10s + timeout: 2s + retries: 3 + networks: + - otel-network + deploy: + resources: + limits: + memory: 512M + reservations: + memory: 256M + + otel-collector: + image: otel/opentelemetry-collector-contrib:latest + restart: unless-stopped + volumes: + - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml + ports: + - "4317:4317" # OTLP gRPC + - "4318:4318" # OTLP HTTP + - "13133:13133" # Health check + - "1777:1777" # pprof + - "55679:55679" # zpages + depends_on: + openobserve: + condition: service_healthy + networks: + - otel-network + deploy: + resources: + limits: + memory: 400M + reservations: + memory: 200M + +networks: + otel-network: + driver: bridge + name: otel-network + ipam: + config: + - subnet: 172.28.0.0/16 + gateway: 172.28.0.1 + labels: + com.example.description: "Network for OpenObserve and OpenTelemetry Collector" +volumes: + data: \ No newline at end of file diff --git a/.docker/openobserve-otel/otel-collector-config.yaml b/.docker/openobserve-otel/otel-collector-config.yaml new file mode 100644 index 00000000..8cdca6f8 --- /dev/null +++ b/.docker/openobserve-otel/otel-collector-config.yaml @@ -0,0 +1,58 @@ +receivers: + otlp: + protocols: + grpc: + endpoint: 0.0.0.0:4317 + http: + endpoint: 0.0.0.0:4318 + filelog: + include: [ "/var/log/app/*.log" ] + start_at: end + +processors: + batch: + timeout: 1s + send_batch_size: 1024 + memory_limiter: + check_interval: 1s + limit_mib: 400 + spike_limit_mib: 100 + +exporters: + otlphttp/openobserve: + endpoint: http://openobserve:5080/api/default + headers: + Authorization: "Bearer cm9vdEBydXN0ZnMuY29tOmxIV0RqQmZMWXJ6MnZOajU=" + stream-name: default + organization: default + tls: + insecure: true + compression: gzip + retry_on_failure: + enabled: true + initial_interval: 5s + max_interval: 30s + max_elapsed_time: 300s + +service: + pipelines: + traces: + receivers: [ otlp ] + processors: [ memory_limiter, batch ] + exporters: [ otlphttp/openobserve ] + metrics: + receivers: [ otlp ] + processors: [ memory_limiter, batch ] + exporters: [ otlphttp/openobserve ] + logs: + receivers: [ otlp, filelog ] + processors: [ memory_limiter, batch ] + exporters: [ otlphttp/openobserve ] + extensions: + health_check: + endpoint: 0.0.0.0:13133 + pprof: + endpoint: 0.0.0.0:1777 + zpages: + endpoint: 0.0.0.0:55679 + diff --git a/scripts/run.sh b/scripts/run.sh index 2004e05a..29605648 100755 --- a/scripts/run.sh +++ b/scripts/run.sh @@ -37,7 +37,7 @@ export RUSTFS_CONSOLE_ADDRESS=":9002" # export RUSTFS_TLS_PATH="./deploy/certs" # 具体路径修改为配置文件真实路径，obs.example.toml 仅供参考 其中 `RUSTFS_OBS_CONFIG` 和下面变量二选一 -export RUSTFS_OBS_CONFIG="./deploy/config/obs.example.toml" +# export RUSTFS_OBS_CONFIG="./deploy/config/obs.example.toml" # 如下变量需要必须参数都有值才可以，以及会覆盖配置文件中的值 #export RUSTFS__OBSERVABILITY__ENDPOINT=http://localhost:4317 From 7dde4083db4fffe28e000684f1e9d407378a95a5 Mon Sep 17 00:00:00 2001 From: houseme Date: Thu, 29 May 2025 09:53:03 +0800 Subject: [PATCH 2/2] upgrade config --- .docker/openobserve-otel/docker-compose.yml | 16 +++++++----- .../otel-collector-config.yaml | 25 +++++++++++-------- .gitignore | 3 ++- 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/.docker/openobserve-otel/docker-compose.yml b/.docker/openobserve-otel/docker-compose.yml index 13d77625..68788aba 100644 --- a/.docker/openobserve-otel/docker-compose.yml +++ b/.docker/openobserve-otel/docker-compose.yml @@ -13,25 +13,29 @@ services: RUST_LOG: "info" ports: - "5080:5080" + - "5081:5081" volumes: - ./data:/data healthcheck: test: [ "CMD", "curl", "-f", "http://localhost:5080/health" ] + start_period: 60s interval: 10s - timeout: 2s - retries: 3 + timeout: 5s + retries: 6 networks: - otel-network deploy: resources: limits: - memory: 512M + memory: 1024M reservations: - memory: 256M + memory: 512M otel-collector: image: otel/opentelemetry-collector-contrib:latest restart: unless-stopped + environment: + - TZ=Asia/Shanghai volumes: - ./otel-collector-config.yaml:/etc/otelcol-contrib/config.yaml ports: @@ -48,9 +52,9 @@ services: deploy: resources: limits: - memory: 400M + memory: 10240M reservations: - memory: 200M + memory: 512M networks: otel-network: diff --git a/.docker/openobserve-otel/otel-collector-config.yaml b/.docker/openobserve-otel/otel-collector-config.yaml index 8cdca6f8..288ac9a2 100644 --- a/.docker/openobserve-otel/otel-collector-config.yaml +++ b/.docker/openobserve-otel/otel-collector-config.yaml @@ -25,16 +25,24 @@ exporters: Authorization: "Bearer cm9vdEBydXN0ZnMuY29tOmxIV0RqQmZMWXJ6MnZOajU=" stream-name: default organization: default - tls: - insecure: true compression: gzip retry_on_failure: enabled: true initial_interval: 5s max_interval: 30s max_elapsed_time: 300s + timeout: 10s + +extensions: + health_check: + endpoint: 0.0.0.0:13133 + pprof: + endpoint: 0.0.0.0:1777 + zpages: + endpoint: 0.0.0.0:55679 service: + extensions: [ health_check, pprof, zpages ] pipelines: traces: receivers: [ otlp ] @@ -48,11 +56,8 @@ service: receivers: [ otlp, filelog ] processors: [ memory_limiter, batch ] exporters: [ otlphttp/openobserve ] - extensions: - health_check: - endpoint: 0.0.0.0:13133 - pprof: - endpoint: 0.0.0.0:1777 - zpages: - endpoint: 0.0.0.0:55679 - + telemetry: + logs: + level: "info" # Collector 日志级别 + metrics: + address: "0.0.0.0:8888" # Collector 自身指标暴露 diff --git a/.gitignore b/.gitignore index 46bfa8d4..d38ffae3 100644 --- a/.gitignore +++ b/.gitignore @@ -16,4 +16,5 @@ deploy/certs/* .env .rustfs.sys .cargo -profile.json \ No newline at end of file +profile.json +.docker/openobserve-otel/data \ No newline at end of file