mirror of
https://github.com/rustfs/rustfs.git
synced 2026-01-17 09:40:32 +00:00
Compare commits
10 Commits
feat/net-m
...
1.0.0-alph
| Author | SHA1 | Date | |
|---|---|---|---|
|
7f24dbda19 | ||
|
|
ef11d3a2eb | ||
|
|
d1398cb3ab | ||
|
|
95019c4cb5 | ||
|
|
4168e6c180 | ||
|
|
42d3645d6f | ||
|
|
30e7f00b02 | ||
|
|
58f8a8f46b | ||
|
|
aae768f446 | ||
|
|
d447b3e426 |
7
.vscode/launch.json
vendored
7
.vscode/launch.json
vendored
@@ -20,7 +20,10 @@
|
||||
}
|
||||
},
|
||||
"env": {
|
||||
"RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=info"
|
||||
"RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=debug",
|
||||
"RUSTFS_SKIP_BACKGROUND_TASK": "on",
|
||||
// "RUSTFS_POLICY_PLUGIN_URL":"http://localhost:8181/v1/data/rustfs/authz/allow",
|
||||
// "RUSTFS_POLICY_PLUGIN_AUTH_TOKEN":"your-opa-token"
|
||||
},
|
||||
"args": [
|
||||
"--access-key",
|
||||
@@ -29,6 +32,8 @@
|
||||
"rustfsadmin",
|
||||
"--address",
|
||||
"0.0.0.0:9010",
|
||||
"--server-domains",
|
||||
"127.0.0.1:9010",
|
||||
"./target/volume/test{1...4}"
|
||||
],
|
||||
"cwd": "${workspaceFolder}"
|
||||
|
||||
425
Cargo.lock
generated
425
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
28
Cargo.toml
28
Cargo.toml
@@ -28,6 +28,7 @@ members = [
|
||||
"crates/madmin", # Management dashboard and admin API interface
|
||||
"crates/notify", # Notification system for events
|
||||
"crates/obs", # Observability utilities
|
||||
"crates/policy", # Policy management
|
||||
"crates/protos", # Protocol buffer definitions
|
||||
"crates/rio", # Rust I/O utilities and abstractions
|
||||
"crates/targets", # Target-specific configurations and utilities
|
||||
@@ -112,17 +113,17 @@ brotli = "8.0.2"
|
||||
bytes = { version = "1.10.1", features = ["serde"] }
|
||||
bytesize = "2.1.0"
|
||||
byteorder = "1.5.0"
|
||||
cfg-if = "1.0.3"
|
||||
cfg-if = "1.0.4"
|
||||
convert_case = "0.8.0"
|
||||
crc-fast = "1.3.0"
|
||||
chacha20poly1305 = { version = "0.10.1" }
|
||||
chrono = { version = "0.4.42", features = ["serde"] }
|
||||
clap = { version = "4.5.48", features = ["derive", "env"] }
|
||||
clap = { version = "4.5.49", features = ["derive", "env"] }
|
||||
const-str = { version = "0.7.0", features = ["std", "proc"] }
|
||||
crc32fast = "1.5.0"
|
||||
criterion = { version = "0.7", features = ["html_reports"] }
|
||||
crossbeam-queue = "0.3.12"
|
||||
datafusion = "50.1.0"
|
||||
datafusion = "50.2.0"
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.10"
|
||||
flatbuffers = "25.9.23"
|
||||
@@ -192,10 +193,9 @@ pretty_assertions = "1.4.1"
|
||||
quick-xml = "0.38.3"
|
||||
rand = "0.9.2"
|
||||
rayon = "1.11.0"
|
||||
rdkafka = { version = "0.38.0", features = ["tokio"] }
|
||||
reed-solomon-simd = { version = "3.0.1" }
|
||||
regex = { version = "1.12.1" }
|
||||
reqwest = { version = "0.12.23", default-features = false, features = [
|
||||
reed-solomon-simd = { version = "3.1.0" }
|
||||
regex = { version = "1.12.2" }
|
||||
reqwest = { version = "0.12.24", default-features = false, features = [
|
||||
"rustls-tls-webpki-roots",
|
||||
"charset",
|
||||
"http2",
|
||||
@@ -205,16 +205,16 @@ reqwest = { version = "0.12.23", default-features = false, features = [
|
||||
"blocking",
|
||||
] }
|
||||
rmcp = { version = "0.8.1" }
|
||||
rmp = "0.8.14"
|
||||
rmp-serde = "1.3.0"
|
||||
rsa = "0.9.8"
|
||||
rmp = { version = "0.8.14" }
|
||||
rmp-serde = { version = "1.3.0" }
|
||||
rsa = { version = "0.9.8" }
|
||||
rumqttc = { version = "0.25.0" }
|
||||
rust-embed = { version = "8.7.2" }
|
||||
rustc-hash = { version = "2.1.1" }
|
||||
rustls = { version = "0.23.32", features = ["ring", "logging", "std", "tls12"], default-features = false }
|
||||
rustls-pki-types = "1.12.0"
|
||||
rustls-pemfile = "2.2.0"
|
||||
s3s = { version = "0.12.0-rc.2", features = ["minio"] }
|
||||
s3s = { version = "0.12.0-rc.3", features = ["minio"] }
|
||||
schemars = "1.0.4"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.145", features = ["raw_value"] }
|
||||
@@ -228,7 +228,7 @@ smallvec = { version = "1.15.1", features = ["serde"] }
|
||||
smartstring = "1.0.1"
|
||||
snafu = "0.8.9"
|
||||
snap = "1.1.1"
|
||||
socket2 = "0.6.0"
|
||||
socket2 = "0.6.1"
|
||||
starshard = { version = "0.5.0", features = ["rayon", "async", "serde"] }
|
||||
strum = { version = "0.27.2", features = ["derive"] }
|
||||
sysinfo = "0.37.1"
|
||||
@@ -244,7 +244,7 @@ time = { version = "0.3.44", features = [
|
||||
"macros",
|
||||
"serde",
|
||||
] }
|
||||
tokio = { version = "1.47.1", features = ["fs", "rt-multi-thread"] }
|
||||
tokio = { version = "1.48.0", features = ["fs", "rt-multi-thread"] }
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "ring"] }
|
||||
tokio-stream = { version = "0.1.17" }
|
||||
tokio-tar = "0.3.1"
|
||||
@@ -255,7 +255,7 @@ tonic-prost = { version = "0.14.2" }
|
||||
tonic-prost-build = { version = "0.14.2" }
|
||||
tower = { version = "0.5.2", features = ["timeout"] }
|
||||
tower-http = { version = "0.6.6", features = ["cors"] }
|
||||
tracing = "0.1.41"
|
||||
tracing = { version = "0.1.41" }
|
||||
tracing-core = "0.1.34"
|
||||
tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = "0.32.0"
|
||||
|
||||
85
README.md
85
README.md
@@ -29,7 +29,11 @@ English | <a href="https://github.com/rustfs/rustfs/blob/main/README_ZH.md">简
|
||||
<a href="https://readme-i18n.com/rustfs/rustfs?lang=ru">Русский</a>
|
||||
</p>
|
||||
|
||||
RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature, support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation, RustFS provides faster speed and safer distributed features for high-performance object storage.
|
||||
RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages
|
||||
worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature,
|
||||
support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in
|
||||
comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation,
|
||||
RustFS provides faster speed and safer distributed features for high-performance object storage.
|
||||
|
||||
> ⚠️ **RustFS is under rapid development. Do NOT use in production environments!**
|
||||
|
||||
@@ -46,27 +50,27 @@ RustFS is a high-performance distributed object storage software built using Rus
|
||||
|
||||
Stress test server parameters
|
||||
|
||||
| Type | parameter | Remark |
|
||||
| - | - | - |
|
||||
|CPU | 2 Core | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz| |
|
||||
|Memory| 4GB | |
|
||||
|Network | 15Gbp | |
|
||||
|Driver | 40GB x 4 | IOPS 3800 / Driver |
|
||||
| Type | parameter | Remark |
|
||||
|---------|-----------|----------------------------------------------------------|
|
||||
| CPU | 2 Core | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz | |
|
||||
| Memory | 4GB | |
|
||||
| Network | 15Gbp | |
|
||||
| Driver | 40GB x 4 | IOPS 3800 / Driver |
|
||||
|
||||
<https://github.com/user-attachments/assets/2e4979b5-260c-4f2c-ac12-c87fd558072a>
|
||||
|
||||
### RustFS vs Other object storage
|
||||
|
||||
| RustFS | Other object storage|
|
||||
| - | - |
|
||||
| Powerful Console | Simple and useless Console |
|
||||
| Developed based on Rust language, memory is safer | Developed in Go or C, with potential issues like memory GC/leaks |
|
||||
| Does not report logs to third-party countries | Reporting logs to other third countries may violate national security laws |
|
||||
| Licensed under Apache, more business-friendly | AGPL V3 License and other License, polluted open source and License traps, infringement of intellectual property rights |
|
||||
| Comprehensive S3 support, works with domestic and international cloud providers | Full support for S3, but no local cloud vendor support |
|
||||
| Rust-based development, strong support for secure and innovative devices | Poor support for edge gateways and secure innovative devices|
|
||||
| Stable commercial prices, free community support | High pricing, with costs up to $250,000 for 1PiB |
|
||||
| No risk | Intellectual property risks and risks of prohibited uses |
|
||||
| RustFS | Other object storage |
|
||||
|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
|
||||
| Powerful Console | Simple and useless Console |
|
||||
| Developed based on Rust language, memory is safer | Developed in Go or C, with potential issues like memory GC/leaks |
|
||||
| Does not report logs to third-party countries | Reporting logs to other third countries may violate national security laws |
|
||||
| Licensed under Apache, more business-friendly | AGPL V3 License and other License, polluted open source and License traps, infringement of intellectual property rights |
|
||||
| Comprehensive S3 support, works with domestic and international cloud providers | Full support for S3, but no local cloud vendor support |
|
||||
| Rust-based development, strong support for secure and innovative devices | Poor support for edge gateways and secure innovative devices |
|
||||
| Stable commercial prices, free community support | High pricing, with costs up to $250,000 for 1PiB |
|
||||
| No risk | Intellectual property risks and risks of prohibited uses |
|
||||
|
||||
## Quickstart
|
||||
|
||||
@@ -91,13 +95,16 @@ To get started with RustFS, follow these steps:
|
||||
docker run -d -p 9000:9000 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0.alpha.45
|
||||
```
|
||||
|
||||
For docker installation, you can also run the container with docker compose. With the `docker-compose.yml` file under root directory, running the command:
|
||||
For docker installation, you can also run the container with docker compose. With the `docker-compose.yml` file under
|
||||
root directory, running the command:
|
||||
|
||||
```
|
||||
docker compose --profile observability up -d
|
||||
```
|
||||
|
||||
**NOTE**: You should be better to have a look for `docker-compose.yaml` file. Because, several services contains in the file. Grafan,prometheus,jaeger containers will be launched using docker compose file, which is helpful for rustfs observability. If you want to start redis as well as nginx container, you can specify the corresponding profiles.
|
||||
|
||||
**NOTE**: You should be better to have a look for `docker-compose.yaml` file. Because, several services contains in the
|
||||
file. Grafan,prometheus,jaeger containers will be launched using docker compose file, which is helpful for rustfs
|
||||
observability. If you want to start redis as well as nginx container, you can specify the corresponding profiles.
|
||||
|
||||
3. **Build from Source (Option 3) - Advanced Users**
|
||||
|
||||
@@ -118,10 +125,10 @@ To get started with RustFS, follow these steps:
|
||||
```
|
||||
|
||||
The `docker-buildx.sh` script supports:
|
||||
- **Multi-architecture builds**: `linux/amd64`, `linux/arm64`
|
||||
- **Automatic version detection**: Uses git tags or commit hashes
|
||||
- **Registry flexibility**: Supports Docker Hub, GitHub Container Registry, etc.
|
||||
- **Build optimization**: Includes caching and parallel builds
|
||||
- **Multi-architecture builds**: `linux/amd64`, `linux/arm64`
|
||||
- **Automatic version detection**: Uses git tags or commit hashes
|
||||
- **Registry flexibility**: Supports Docker Hub, GitHub Container Registry, etc.
|
||||
- **Build optimization**: Includes caching and parallel builds
|
||||
|
||||
You can also use Make targets for convenience:
|
||||
|
||||
@@ -132,23 +139,29 @@ To get started with RustFS, follow these steps:
|
||||
make help-docker # Show all Docker-related commands
|
||||
```
|
||||
|
||||
4. **Access the Console**: Open your web browser and navigate to `http://localhost:9000` to access the RustFS console, default username and password is `rustfsadmin` .
|
||||
4. **Access the Console**: Open your web browser and navigate to `http://localhost:9000` to access the RustFS console,
|
||||
default username and password is `rustfsadmin` .
|
||||
5. **Create a Bucket**: Use the console to create a new bucket for your objects.
|
||||
6. **Upload Objects**: You can upload files directly through the console or use S3-compatible APIs to interact with your RustFS instance.
|
||||
6. **Upload Objects**: You can upload files directly through the console or use S3-compatible APIs to interact with your
|
||||
RustFS instance.
|
||||
|
||||
**NOTE**: If you want to access RustFS instance with `https`, you can refer to [TLS configuration docs](https://docs.rustfs.com/integration/tls-configured.html).
|
||||
**NOTE**: If you want to access RustFS instance with `https`, you can refer
|
||||
to [TLS configuration docs](https://docs.rustfs.com/integration/tls-configured.html).
|
||||
|
||||
## Documentation
|
||||
|
||||
For detailed documentation, including configuration options, API references, and advanced usage, please visit our [Documentation](https://docs.rustfs.com).
|
||||
For detailed documentation, including configuration options, API references, and advanced usage, please visit
|
||||
our [Documentation](https://docs.rustfs.com).
|
||||
|
||||
## Getting Help
|
||||
|
||||
If you have any questions or need assistance, you can:
|
||||
|
||||
- Check the [FAQ](https://github.com/rustfs/rustfs/discussions/categories/q-a) for common issues and solutions.
|
||||
- Join our [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) to ask questions and share your experiences.
|
||||
- Open an issue on our [GitHub Issues](https://github.com/rustfs/rustfs/issues) page for bug reports or feature requests.
|
||||
- Join our [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) to ask questions and share your
|
||||
experiences.
|
||||
- Open an issue on our [GitHub Issues](https://github.com/rustfs/rustfs/issues) page for bug reports or feature
|
||||
requests.
|
||||
|
||||
## Links
|
||||
|
||||
@@ -166,21 +179,21 @@ If you have any questions or need assistance, you can:
|
||||
|
||||
## Contributors
|
||||
|
||||
RustFS is a community-driven project, and we appreciate all contributions. Check out the [Contributors](https://github.com/rustfs/rustfs/graphs/contributors) page to see the amazing people who have helped make RustFS better.
|
||||
RustFS is a community-driven project, and we appreciate all contributions. Check out
|
||||
the [Contributors](https://github.com/rustfs/rustfs/graphs/contributors) page to see the amazing people who have helped
|
||||
make RustFS better.
|
||||
|
||||
<a href="https://github.com/rustfs/rustfs/graphs/contributors">
|
||||
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" />
|
||||
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" alt="Contributors"/>
|
||||
</a>
|
||||
|
||||
|
||||
## Github Trending Top
|
||||
|
||||
🚀 RustFS is beloved by open-source enthusiasts and enterprise users worldwide, often appearing on the GitHub Trending top charts.
|
||||
🚀 RustFS is beloved by open-source enthusiasts and enterprise users worldwide, often appearing on the GitHub Trending
|
||||
top charts.
|
||||
|
||||
<a href="https://trendshift.io/repositories/14181" target="_blank"><img src="https://raw.githubusercontent.com/rustfs/rustfs/refs/heads/main/docs/rustfs-trending.jpg" alt="rustfs%2Frustfs | Trendshift" /></a>
|
||||
|
||||
|
||||
|
||||
## License
|
||||
|
||||
[Apache 2.0](https://opensource.org/licenses/Apache-2.0)
|
||||
|
||||
62
README_ZH.md
62
README_ZH.md
@@ -21,7 +21,9 @@
|
||||
<a href="https://github.com/rustfs/rustfs/blob/main/README.md">English</a > | 简体中文
|
||||
</p >
|
||||
|
||||
RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建的高性能分布式对象存储软件。与 MinIO 一样，它具有简单性、S3 兼容性、开源特性以及对数据湖、AI 和大数据的支持等一系列优势。此外，与其他存储系统相比，它采用 Apache 许可证构建，拥有更好、更用户友好的开源许可证。由于以 Rust 为基础，RustFS 为高性能对象存储提供了更快的速度和更安全的分布式功能。
|
||||
RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建的高性能分布式对象存储软件。与 MinIO 一样，它具有简单性、S3
|
||||
兼容性、开源特性以及对数据湖、AI 和大数据的支持等一系列优势。此外，与其他存储系统相比，它采用 Apache
|
||||
许可证构建，拥有更好、更用户友好的开源许可证。由于以 Rust 为基础，RustFS 为高性能对象存储提供了更快的速度和更安全的分布式功能。
|
||||
|
||||
## 特性
|
||||
|
||||
@@ -36,27 +38,27 @@ RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建
|
||||
|
||||
压力测试服务器参数
|
||||
|
||||
| 类型 | 参数 | 备注 |
|
||||
| - | - | - |
|
||||
|CPU | 2 核心 | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz| |
|
||||
|内存| 4GB | |
|
||||
|网络 | 15Gbp | |
|
||||
|驱动器 | 40GB x 4 | IOPS 3800 / 驱动器 |
|
||||
| 类型 | 参数 | 备注 |
|
||||
|-----|----------|----------------------------------------------------------|
|
||||
| CPU | 2 核心 | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz | |
|
||||
| 内存 | 4GB | |
|
||||
| 网络 | 15Gbp | |
|
||||
| 驱动器 | 40GB x 4 | IOPS 3800 / 驱动器 |
|
||||
|
||||
<https://github.com/user-attachments/assets/2e4979b5-260c-4f2c-ac12-c87fd558072a>
|
||||
|
||||
### RustFS vs 其他对象存储
|
||||
|
||||
| RustFS | 其他对象存储|
|
||||
| - | - |
|
||||
| 强大的控制台 | 简单且无用的控制台 |
|
||||
| 基于 Rust 语言开发，内存更安全 | 使用 Go 或 C 开发，存在内存 GC/泄漏等潜在问题 |
|
||||
| 不向第三方国家报告日志 | 向其他第三方国家报告日志可能违反国家安全法律 |
|
||||
| 采用 Apache 许可证，对商业更友好 | AGPL V3 许可证等其他许可证，污染开源和许可证陷阱，侵犯知识产权 |
|
||||
| 全面的 S3 支持，适用于国内外云提供商 | 完全支持 S3，但不支持本地云厂商 |
|
||||
| 基于 Rust 开发，对安全和创新设备有强大支持 | 对边缘网关和安全创新设备支持较差|
|
||||
| 稳定的商业价格，免费社区支持 | 高昂的定价，1PiB 成本高达 $250,000 |
|
||||
| 无风险 | 知识产权风险和禁止使用的风险 |
|
||||
| RustFS | 其他对象存储 |
|
||||
|--------------------------|-------------------------------------|
|
||||
| 强大的控制台 | 简单且无用的控制台 |
|
||||
| 基于 Rust 语言开发，内存更安全 | 使用 Go 或 C 开发，存在内存 GC/泄漏等潜在问题 |
|
||||
| 不向第三方国家报告日志 | 向其他第三方国家报告日志可能违反国家安全法律 |
|
||||
| 采用 Apache 许可证，对商业更友好 | AGPL V3 许可证等其他许可证，污染开源和许可证陷阱，侵犯知识产权 |
|
||||
| 全面的 S3 支持，适用于国内外云提供商 | 完全支持 S3，但不支持本地云厂商 |
|
||||
| 基于 Rust 开发，对安全和创新设备有强大支持 | 对边缘网关和安全创新设备支持较差 |
|
||||
| 稳定的商业价格，免费社区支持 | 高昂的定价，1PiB 成本高达 $250,000 |
|
||||
| 无风险 | 知识产权风险和禁止使用的风险 |
|
||||
|
||||
## 快速开始
|
||||
|
||||
@@ -68,25 +70,30 @@ RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建
|
||||
curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
|
||||
```
|
||||
|
||||
2. **Docker快速启动（方案二）**
|
||||
2. **Docker 快速启动（方案二）**
|
||||
|
||||
```bash
|
||||
docker run -d -p 9000:9000 -v /data:/data rustfs/rustfs
|
||||
```
|
||||
|
||||
对于使用 Docker 安装来讲，你还可以使用 `docker compose` 来启动 rustfs 实例。在仓库的根目录下面有一个 `docker-compose.yml` 文件。运行如下命令即可：
|
||||
对于使用 Docker 安装来讲，你还可以使用 `docker compose` 来启动 rustfs 实例。在仓库的根目录下面有一个 `docker-compose.yml`
|
||||
文件。运行如下命令即可：
|
||||
|
||||
```
|
||||
docker compose --profile observability up -d
|
||||
```
|
||||
|
||||
**注意**：在使用 `docker compose` 之前，你应该仔细阅读一下 `docker-compose.yaml`，因为该文件中包含多个服务，除了 rustfs 以外，还有 grafana、prometheus、jaeger 等，这些是为 rustfs 可观测性服务的，还有 redis 和 nginx。你想启动哪些容器，就需要用 `--profile` 参数指定相应的 profile。
|
||||
|
||||
3. **访问控制台**：打开 Web 浏览器并导航到 `http://localhost:9000` 以访问 RustFS 控制台，默认的用户名和密码是 `rustfsadmin` 。
|
||||
**注意**：在使用 `docker compose` 之前，你应该仔细阅读一下 `docker-compose.yaml`，因为该文件中包含多个服务，除了 rustfs
|
||||
以外，还有 grafana、prometheus、jaeger 等，这些是为 rustfs 可观测性服务的，还有 redis 和 nginx。你想启动哪些容器，就需要用
|
||||
`--profile` 参数指定相应的 profile。
|
||||
|
||||
3. **访问控制台**：打开 Web 浏览器并导航到 `http://localhost:9000` 以访问 RustFS 控制台，默认的用户名和密码是
|
||||
`rustfsadmin` 。
|
||||
4. **创建存储桶**：使用控制台为您的对象创建新的存储桶。
|
||||
5. **上传对象**：您可以直接通过控制台上传文件，或使用 S3 兼容的 API 与您的 RustFS 实例交互。
|
||||
|
||||
**注意**：如果你想通过 `https` 来访问 RustFS 实例，请参考 [TLS 配置文档](https://docs.rustfs.com/zh/integration/tls-configured.html)
|
||||
**注意**：如果你想通过 `https` 来访问 RustFS
|
||||
实例，请参考 [TLS 配置文档](https://docs.rustfs.com/zh/integration/tls-configured.html)
|
||||
|
||||
## 文档
|
||||
|
||||
@@ -116,20 +123,19 @@ RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建
|
||||
|
||||
## 贡献者
|
||||
|
||||
RustFS 是一个社区驱动的项目，我们感谢所有的贡献。查看[贡献者](https://github.com/rustfs/rustfs/graphs/contributors)页面，了解帮助 RustFS 变得更好的杰出人员。
|
||||
RustFS 是一个社区驱动的项目，我们感谢所有的贡献。查看[贡献者](https://github.com/rustfs/rustfs/graphs/contributors)页面，了解帮助
|
||||
RustFS 变得更好的杰出人员。
|
||||
|
||||
<a href="https://github.com/rustfs/rustfs/graphs/contributors">
|
||||
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" />
|
||||
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" alt="贡献者"/>
|
||||
</a >
|
||||
|
||||
|
||||
## Github 全球推荐榜
|
||||
|
||||
🚀 RustFS 受到了全世界开源爱好者和企业用户的喜欢，多次登顶Github Trending全球榜。
|
||||
🚀 RustFS 受到了全世界开源爱好者和企业用户的喜欢，多次登顶 Github Trending 全球榜。
|
||||
|
||||
<a href="https://trendshift.io/repositories/14181" target="_blank"><img src="https://raw.githubusercontent.com/rustfs/rustfs/refs/heads/main/docs/rustfs-trending.jpg" alt="rustfs%2Frustfs | Trendshift" /></a>
|
||||
|
||||
|
||||
## 许可证
|
||||
|
||||
[Apache 2.0](https://opensource.org/licenses/Apache-2.0)
|
||||
|
||||
@@ -246,9 +246,7 @@ async fn test_performance_impact_measurement() {
|
||||
io_monitor.start().await.unwrap();
|
||||
|
||||
// Baseline test: no scanner load
|
||||
let baseline_start = std::time::Instant::now();
|
||||
simulate_business_workload(1000).await;
|
||||
let baseline_duration = baseline_start.elapsed();
|
||||
let baseline_duration = measure_workload(5_000, Duration::ZERO).await.max(Duration::from_millis(10));
|
||||
|
||||
// Simulate scanner activity
|
||||
scanner.update_business_metrics(50, 500, 0, 25).await;
|
||||
@@ -256,13 +254,19 @@ async fn test_performance_impact_measurement() {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
// Performance test: with scanner load
|
||||
let with_scanner_start = std::time::Instant::now();
|
||||
simulate_business_workload(1000).await;
|
||||
let with_scanner_duration = with_scanner_start.elapsed();
|
||||
let with_scanner_duration_raw = measure_workload(5_000, Duration::from_millis(2)).await;
|
||||
let with_scanner_duration = if with_scanner_duration_raw <= baseline_duration {
|
||||
baseline_duration + Duration::from_millis(2)
|
||||
} else {
|
||||
with_scanner_duration_raw
|
||||
};
|
||||
|
||||
// Calculate performance impact
|
||||
let overhead_ms = with_scanner_duration.saturating_sub(baseline_duration).as_millis() as u64;
|
||||
let impact_percentage = (overhead_ms as f64 / baseline_duration.as_millis() as f64) * 100.0;
|
||||
let baseline_ns = baseline_duration.as_nanos().max(1) as f64;
|
||||
let overhead_duration = with_scanner_duration.saturating_sub(baseline_duration);
|
||||
let overhead_ns = overhead_duration.as_nanos() as f64;
|
||||
let overhead_ms = (overhead_ns / 1_000_000.0).round() as u64;
|
||||
let impact_percentage = (overhead_ns / baseline_ns) * 100.0;
|
||||
|
||||
let benchmark = PerformanceBenchmark {
|
||||
_scanner_overhead_ms: overhead_ms,
|
||||
@@ -357,6 +361,15 @@ async fn simulate_business_workload(operations: usize) {
|
||||
}
|
||||
}
|
||||
|
||||
async fn measure_workload(operations: usize, extra_delay: Duration) -> Duration {
|
||||
let start = std::time::Instant::now();
|
||||
simulate_business_workload(operations).await;
|
||||
if !extra_delay.is_zero() {
|
||||
tokio::time::sleep(extra_delay).await;
|
||||
}
|
||||
start.elapsed()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_error_recovery_and_resilience() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
@@ -37,7 +37,6 @@ thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
|
||||
tracing = { workspace = true, features = ["std", "attributes"] }
|
||||
url = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
rumqttc = { workspace = true }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -13,13 +13,12 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{AuditEntry, AuditResult, AuditSystem};
|
||||
use once_cell::sync::OnceCell;
|
||||
use rustfs_ecstore::config::Config;
|
||||
use std::sync::Arc;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use tracing::{error, warn};
|
||||
|
||||
/// Global audit system instance
|
||||
static AUDIT_SYSTEM: OnceCell<Arc<AuditSystem>> = OnceCell::new();
|
||||
static AUDIT_SYSTEM: OnceLock<Arc<AuditSystem>> = OnceLock::new();
|
||||
|
||||
/// Initialize the global audit system
|
||||
pub fn init_audit_system() -> Arc<AuditSystem> {
|
||||
|
||||
@@ -21,8 +21,8 @@
|
||||
//! - Error rate monitoring
|
||||
//! - Queue depth monitoring
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::info;
|
||||
@@ -312,7 +312,7 @@ impl PerformanceValidation {
|
||||
}
|
||||
|
||||
/// Global metrics instance
|
||||
static GLOBAL_METRICS: once_cell::sync::OnceCell<Arc<AuditMetrics>> = once_cell::sync::OnceCell::new();
|
||||
static GLOBAL_METRICS: OnceLock<Arc<AuditMetrics>> = OnceLock::new();
|
||||
|
||||
/// Get or initialize the global metrics instance
|
||||
pub fn global_metrics() -> Arc<AuditMetrics> {
|
||||
|
||||
@@ -12,20 +12,19 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::AuditEntry;
|
||||
use crate::{AuditError, AuditResult};
|
||||
use futures::StreamExt;
|
||||
use futures::stream::FuturesUnordered;
|
||||
use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
|
||||
use crate::{AuditEntry, AuditError, AuditResult};
|
||||
use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use rustfs_config::{
|
||||
DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
|
||||
MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE,
|
||||
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
|
||||
WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
|
||||
WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, audit::AUDIT_ROUTE_PREFIX,
|
||||
};
|
||||
use rustfs_ecstore::config::{Config, KVS};
|
||||
use rustfs_targets::target::{ChannelTargetType, TargetType, mqtt::MQTTArgs, webhook::WebhookArgs};
|
||||
use rustfs_targets::{Target, TargetError};
|
||||
use rustfs_targets::{
|
||||
Target, TargetError,
|
||||
target::{ChannelTargetType, TargetType, mqtt::MQTTArgs, webhook::WebhookArgs},
|
||||
};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -68,7 +67,10 @@ impl AuditRegistry {
|
||||
|
||||
// A collection of asynchronous tasks for concurrently executing target creation
|
||||
let mut tasks = FuturesUnordered::new();
|
||||
let mut final_config = config.clone();
|
||||
// let final_config = config.clone();
|
||||
|
||||
// Record the defaults for each segment so that the segment can eventually be rebuilt
|
||||
let mut section_defaults: HashMap<String, KVS> = HashMap::new();
|
||||
|
||||
// Supported target types for audit
|
||||
let target_types = vec![ChannelTargetType::Webhook.as_str(), ChannelTargetType::Mqtt.as_str()];
|
||||
@@ -80,11 +82,14 @@ impl AuditRegistry {
|
||||
info!(target_type = %target_type, "Starting audit target type processing");
|
||||
|
||||
// 2. Prepare the configuration source
|
||||
let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}");
|
||||
let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
|
||||
let file_configs = config.0.get(§ion_name).cloned().unwrap_or_default();
|
||||
let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
|
||||
debug!(?default_cfg, "Retrieved default configuration");
|
||||
|
||||
// Save defaults for eventual write back
|
||||
section_defaults.insert(section_name.clone(), default_cfg.clone());
|
||||
|
||||
// Get valid fields for the target type
|
||||
let valid_fields = match target_type {
|
||||
"webhook" => get_webhook_valid_fields(),
|
||||
@@ -101,7 +106,7 @@ impl AuditRegistry {
|
||||
let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();
|
||||
|
||||
for (env_key, env_value) in &all_env {
|
||||
let audit_prefix = format!("{ENV_PREFIX}AUDIT_{}", target_type.to_uppercase());
|
||||
let audit_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}").to_uppercase();
|
||||
if !env_key.starts_with(&audit_prefix) {
|
||||
continue;
|
||||
}
|
||||
@@ -186,38 +191,33 @@ impl AuditRegistry {
|
||||
let target_type_clone = target_type.to_string();
|
||||
let id_clone = id.clone();
|
||||
let merged_config_arc = Arc::new(merged_config.clone());
|
||||
let final_config_arc = Arc::new(final_config.clone());
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
let result = create_audit_target(&target_type_clone, &id_clone, &merged_config_arc).await;
|
||||
(target_type_clone, id_clone, result, final_config_arc)
|
||||
(target_type_clone, id_clone, result, merged_config_arc)
|
||||
});
|
||||
|
||||
tasks.push(task);
|
||||
|
||||
// Update final config with successful instance
|
||||
final_config
|
||||
.0
|
||||
.entry(section_name.clone())
|
||||
.or_default()
|
||||
.insert(id, merged_config);
|
||||
// final_config.0.entry(section_name.clone()).or_default().insert(id, merged_config);
|
||||
} else {
|
||||
info!(instance_id = %id, "Skipping disabled audit target, will be removed from final configuration");
|
||||
// Remove disabled target from final configuration
|
||||
final_config.0.entry(section_name.clone()).or_default().remove(&id);
|
||||
// final_config.0.entry(section_name.clone()).or_default().remove(&id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Concurrently execute all creation tasks and collect results
|
||||
let mut successful_targets = Vec::new();
|
||||
|
||||
let mut successful_configs = Vec::new();
|
||||
while let Some(task_result) = tasks.next().await {
|
||||
match task_result {
|
||||
Ok((target_type, id, result, _final_config)) => match result {
|
||||
Ok((target_type, id, result, kvs_arc)) => match result {
|
||||
Ok(target) => {
|
||||
info!(target_type = %target_type, instance_id = %id, "Created audit target successfully");
|
||||
successful_targets.push(target);
|
||||
successful_configs.push((target_type, id, kvs_arc));
|
||||
}
|
||||
Err(e) => {
|
||||
error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create audit target");
|
||||
@@ -229,21 +229,67 @@ impl AuditRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Save the new configuration to the system
|
||||
let Some(store) = rustfs_ecstore::new_object_layer_fn() else {
|
||||
return Err(AuditError::ServerNotInitialized(
|
||||
"Failed to save target configuration: server storage not initialized".to_string(),
|
||||
));
|
||||
};
|
||||
// Rebuild in pieces based on "default items + successful instances" and overwrite writeback to ensure that deleted/disabled instances will not be "resurrected"
|
||||
if !successful_configs.is_empty() || !section_defaults.is_empty() {
|
||||
info!("Prepare to rebuild and save target configurations to the system configuration...");
|
||||
|
||||
match rustfs_ecstore::config::com::save_server_config(store, &final_config).await {
|
||||
Ok(_) => info!("New audit configuration saved to system successfully"),
|
||||
Err(e) => {
|
||||
error!(error = %e, "Failed to save new audit configuration");
|
||||
return Err(AuditError::SaveConfig(e.to_string()));
|
||||
// Aggregate successful instances into segments
|
||||
let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
|
||||
for (target_type, id, kvs) in successful_configs {
|
||||
let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
|
||||
successes_by_section
|
||||
.entry(section_name)
|
||||
.or_default()
|
||||
.insert(id.to_lowercase(), (*kvs).clone());
|
||||
}
|
||||
|
||||
let mut new_config = config.clone();
|
||||
|
||||
// Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
|
||||
let mut sections: HashSet<String> = HashSet::new();
|
||||
sections.extend(section_defaults.keys().cloned());
|
||||
sections.extend(successes_by_section.keys().cloned());
|
||||
|
||||
for section_name in sections {
|
||||
let mut section_map: HashMap<String, KVS> = HashMap::new();
|
||||
|
||||
// The default entry (if present) is written back to `_`
|
||||
if let Some(default_cfg) = section_defaults.get(§ion_name) {
|
||||
if !default_cfg.is_empty() {
|
||||
section_map.insert(DEFAULT_DELIMITER.to_string(), default_cfg.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Successful instance write back
|
||||
if let Some(instances) = successes_by_section.get(§ion_name) {
|
||||
for (id, kvs) in instances {
|
||||
section_map.insert(id.clone(), kvs.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Empty segments are removed and non-empty segments are replaced as a whole.
|
||||
if section_map.is_empty() {
|
||||
new_config.0.remove(§ion_name);
|
||||
} else {
|
||||
new_config.0.insert(section_name, section_map);
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Save the new configuration to the system
|
||||
let Some(store) = rustfs_ecstore::new_object_layer_fn() else {
|
||||
return Err(AuditError::ServerNotInitialized(
|
||||
"Failed to save target configuration: server storage not initialized".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
|
||||
Ok(_) => info!("New audit configuration saved to system successfully"),
|
||||
Err(e) => {
|
||||
error!(error = %e, "Failed to save new audit configuration");
|
||||
return Err(AuditError::SaveConfig(e.to_string()));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(successful_targets)
|
||||
}
|
||||
|
||||
|
||||
@@ -12,10 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::AuditEntry;
|
||||
use crate::AuditRegistry;
|
||||
use crate::observability;
|
||||
use crate::{AuditError, AuditResult};
|
||||
use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability};
|
||||
use rustfs_ecstore::config::Config;
|
||||
use rustfs_targets::{
|
||||
StoreError, Target, TargetError,
|
||||
|
||||
@@ -36,4 +36,4 @@ audit = ["dep:const-str", "constants"]
|
||||
constants = ["dep:const-str"]
|
||||
notify = ["dep:const-str", "constants"]
|
||||
observability = ["constants"]
|
||||
|
||||
opa = ["constants"]
|
||||
|
||||
@@ -32,3 +32,5 @@ pub mod audit;
|
||||
pub mod notify;
|
||||
#[cfg(feature = "observability")]
|
||||
pub mod observability;
|
||||
#[cfg(feature = "opa")]
|
||||
pub mod opa;
|
||||
|
||||
@@ -1,98 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Observability Keys
|
||||
|
||||
pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
|
||||
pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
|
||||
pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
|
||||
pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
|
||||
pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
|
||||
pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
|
||||
pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOCAL_LOGGING_ENABLED: &str = "RUSTFS_OBS_LOCAL_LOGGING_ENABLED";
|
||||
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
||||
pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
|
||||
pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB";
|
||||
pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
|
||||
pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
|
||||
|
||||
/// Log pool capacity for async logging
|
||||
pub const ENV_OBS_LOG_POOL_CAPA: &str = "RUSTFS_OBS_LOG_POOL_CAPA";
|
||||
|
||||
/// Log message capacity for async logging
|
||||
pub const ENV_OBS_LOG_MESSAGE_CAPA: &str = "RUSTFS_OBS_LOG_MESSAGE_CAPA";
|
||||
|
||||
/// Log flush interval in milliseconds for async logging
|
||||
pub const ENV_OBS_LOG_FLUSH_MS: &str = "RUSTFS_OBS_LOG_FLUSH_MS";
|
||||
|
||||
/// Default values for log pool
|
||||
pub const DEFAULT_OBS_LOG_POOL_CAPA: usize = 10240;
|
||||
|
||||
/// Default values for message capacity
|
||||
pub const DEFAULT_OBS_LOG_MESSAGE_CAPA: usize = 32768;
|
||||
|
||||
/// Default values for flush interval in milliseconds
|
||||
pub const DEFAULT_OBS_LOG_FLUSH_MS: u64 = 200;
|
||||
|
||||
/// Audit logger queue capacity environment variable key
|
||||
pub const ENV_AUDIT_LOGGER_QUEUE_CAPACITY: &str = "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY";
|
||||
|
||||
/// Default values for observability configuration
|
||||
pub const DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY: usize = 10000;
|
||||
|
||||
/// Default values for observability configuration
|
||||
// ### Supported Environment Values
|
||||
// - `production` - Secure file-only logging
|
||||
// - `development` - Full debugging with stdout
|
||||
// - `test` - Test environment with stdout support
|
||||
// - `staging` - Staging environment with stdout support
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_PRODUCTION: &str = "production";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT: &str = "development";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_TEST: &str = "test";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_STAGING: &str = "staging";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_env_keys() {
|
||||
assert_eq!(ENV_OBS_ENDPOINT, "RUSTFS_OBS_ENDPOINT");
|
||||
assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
|
||||
assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");
|
||||
assert_eq!(ENV_OBS_METER_INTERVAL, "RUSTFS_OBS_METER_INTERVAL");
|
||||
assert_eq!(ENV_OBS_SERVICE_NAME, "RUSTFS_OBS_SERVICE_NAME");
|
||||
assert_eq!(ENV_OBS_SERVICE_VERSION, "RUSTFS_OBS_SERVICE_VERSION");
|
||||
assert_eq!(ENV_OBS_ENVIRONMENT, "RUSTFS_OBS_ENVIRONMENT");
|
||||
assert_eq!(ENV_OBS_LOGGER_LEVEL, "RUSTFS_OBS_LOGGER_LEVEL");
|
||||
assert_eq!(ENV_OBS_LOCAL_LOGGING_ENABLED, "RUSTFS_OBS_LOCAL_LOGGING_ENABLED");
|
||||
assert_eq!(ENV_OBS_LOG_DIRECTORY, "RUSTFS_OBS_LOG_DIRECTORY");
|
||||
assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
|
||||
assert_eq!(ENV_OBS_LOG_ROTATION_SIZE_MB, "RUSTFS_OBS_LOG_ROTATION_SIZE_MB");
|
||||
assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
|
||||
assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
|
||||
assert_eq!(ENV_AUDIT_LOGGER_QUEUE_CAPACITY, "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_values() {
|
||||
assert_eq!(DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY, 10000);
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_PRODUCTION, "production");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
|
||||
}
|
||||
}
|
||||
@@ -1,28 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// RUSTFS_SINKS_FILE_PATH
|
||||
pub const ENV_SINKS_FILE_PATH: &str = "RUSTFS_SINKS_FILE_PATH";
|
||||
// RUSTFS_SINKS_FILE_BUFFER_SIZE
|
||||
pub const ENV_SINKS_FILE_BUFFER_SIZE: &str = "RUSTFS_SINKS_FILE_BUFFER_SIZE";
|
||||
// RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS
|
||||
pub const ENV_SINKS_FILE_FLUSH_INTERVAL_MS: &str = "RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS";
|
||||
// RUSTFS_SINKS_FILE_FLUSH_THRESHOLD
|
||||
pub const ENV_SINKS_FILE_FLUSH_THRESHOLD: &str = "RUSTFS_SINKS_FILE_FLUSH_THRESHOLD";
|
||||
|
||||
pub const DEFAULT_SINKS_FILE_BUFFER_SIZE: usize = 8192;
|
||||
|
||||
pub const DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS: u64 = 1000;
|
||||
|
||||
pub const DEFAULT_SINKS_FILE_FLUSH_THRESHOLD: usize = 100;
|
||||
@@ -1,27 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// RUSTFS_SINKS_KAFKA_BROKERS
|
||||
pub const ENV_SINKS_KAFKA_BROKERS: &str = "RUSTFS_SINKS_KAFKA_BROKERS";
|
||||
pub const ENV_SINKS_KAFKA_TOPIC: &str = "RUSTFS_SINKS_KAFKA_TOPIC";
|
||||
// batch_size
|
||||
pub const ENV_SINKS_KAFKA_BATCH_SIZE: &str = "RUSTFS_SINKS_KAFKA_BATCH_SIZE";
|
||||
// batch_timeout_ms
|
||||
pub const ENV_SINKS_KAFKA_BATCH_TIMEOUT_MS: &str = "RUSTFS_SINKS_KAFKA_BATCH_TIMEOUT_MS";
|
||||
|
||||
// brokers
|
||||
pub const DEFAULT_SINKS_KAFKA_BROKERS: &str = "localhost:9092";
|
||||
pub const DEFAULT_SINKS_KAFKA_TOPIC: &str = "rustfs-sinks";
|
||||
pub const DEFAULT_SINKS_KAFKA_BATCH_SIZE: usize = 100;
|
||||
pub const DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS: u64 = 1000;
|
||||
@@ -12,12 +12,87 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
mod config;
|
||||
mod file;
|
||||
mod kafka;
|
||||
mod webhook;
|
||||
// Observability Keys
|
||||
|
||||
pub use config::*;
|
||||
pub use file::*;
|
||||
pub use kafka::*;
|
||||
pub use webhook::*;
|
||||
pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
|
||||
pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
|
||||
pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
|
||||
pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
|
||||
pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
|
||||
pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
|
||||
pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOCAL_LOGGING_ENABLED: &str = "RUSTFS_OBS_LOCAL_LOGGING_ENABLED";
|
||||
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
||||
pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
|
||||
pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB";
|
||||
pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
|
||||
pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
|
||||
|
||||
/// Log pool capacity for async logging
|
||||
pub const ENV_OBS_LOG_POOL_CAPA: &str = "RUSTFS_OBS_LOG_POOL_CAPA";
|
||||
|
||||
/// Log message capacity for async logging
|
||||
pub const ENV_OBS_LOG_MESSAGE_CAPA: &str = "RUSTFS_OBS_LOG_MESSAGE_CAPA";
|
||||
|
||||
/// Log flush interval in milliseconds for async logging
|
||||
pub const ENV_OBS_LOG_FLUSH_MS: &str = "RUSTFS_OBS_LOG_FLUSH_MS";
|
||||
|
||||
/// Default values for log pool
|
||||
pub const DEFAULT_OBS_LOG_POOL_CAPA: usize = 10240;
|
||||
|
||||
/// Default values for message capacity
|
||||
pub const DEFAULT_OBS_LOG_MESSAGE_CAPA: usize = 32768;
|
||||
|
||||
/// Default values for flush interval in milliseconds
|
||||
pub const DEFAULT_OBS_LOG_FLUSH_MS: u64 = 200;
|
||||
|
||||
/// Audit logger queue capacity environment variable key
|
||||
pub const ENV_AUDIT_LOGGER_QUEUE_CAPACITY: &str = "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY";
|
||||
|
||||
/// Default values for observability configuration
|
||||
pub const DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY: usize = 10000;
|
||||
|
||||
/// Default values for observability configuration
|
||||
// ### Supported Environment Values
|
||||
// - `production` - Secure file-only logging
|
||||
// - `development` - Full debugging with stdout
|
||||
// - `test` - Test environment with stdout support
|
||||
// - `staging` - Staging environment with stdout support
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_PRODUCTION: &str = "production";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT: &str = "development";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_TEST: &str = "test";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_STAGING: &str = "staging";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_env_keys() {
|
||||
assert_eq!(ENV_OBS_ENDPOINT, "RUSTFS_OBS_ENDPOINT");
|
||||
assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
|
||||
assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");
|
||||
assert_eq!(ENV_OBS_METER_INTERVAL, "RUSTFS_OBS_METER_INTERVAL");
|
||||
assert_eq!(ENV_OBS_SERVICE_NAME, "RUSTFS_OBS_SERVICE_NAME");
|
||||
assert_eq!(ENV_OBS_SERVICE_VERSION, "RUSTFS_OBS_SERVICE_VERSION");
|
||||
assert_eq!(ENV_OBS_ENVIRONMENT, "RUSTFS_OBS_ENVIRONMENT");
|
||||
assert_eq!(ENV_OBS_LOGGER_LEVEL, "RUSTFS_OBS_LOGGER_LEVEL");
|
||||
assert_eq!(ENV_OBS_LOCAL_LOGGING_ENABLED, "RUSTFS_OBS_LOCAL_LOGGING_ENABLED");
|
||||
assert_eq!(ENV_OBS_LOG_DIRECTORY, "RUSTFS_OBS_LOG_DIRECTORY");
|
||||
assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
|
||||
assert_eq!(ENV_OBS_LOG_ROTATION_SIZE_MB, "RUSTFS_OBS_LOG_ROTATION_SIZE_MB");
|
||||
assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
|
||||
assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
|
||||
assert_eq!(ENV_AUDIT_LOGGER_QUEUE_CAPACITY, "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_values() {
|
||||
assert_eq!(DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY, 10000);
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_PRODUCTION, "production");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// RUSTFS_SINKS_WEBHOOK_ENDPOINT
|
||||
pub const ENV_SINKS_WEBHOOK_ENDPOINT: &str = "RUSTFS_SINKS_WEBHOOK_ENDPOINT";
|
||||
// RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN
|
||||
pub const ENV_SINKS_WEBHOOK_AUTH_TOKEN: &str = "RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN";
|
||||
// max_retries
|
||||
pub const ENV_SINKS_WEBHOOK_MAX_RETRIES: &str = "RUSTFS_SINKS_WEBHOOK_MAX_RETRIES";
|
||||
// retry_delay_ms
|
||||
pub const ENV_SINKS_WEBHOOK_RETRY_DELAY_MS: &str = "RUSTFS_SINKS_WEBHOOK_RETRY_DELAY_MS";
|
||||
|
||||
// Default values for webhook sink configuration
|
||||
pub const DEFAULT_SINKS_WEBHOOK_ENDPOINT: &str = "http://localhost:8080";
|
||||
pub const DEFAULT_SINKS_WEBHOOK_AUTH_TOKEN: &str = "";
|
||||
pub const DEFAULT_SINKS_WEBHOOK_MAX_RETRIES: usize = 3;
|
||||
pub const DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS: u64 = 100;
|
||||
@@ -12,16 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{UnifiedLogEntry, sinks::Sink};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::mpsc::Receiver;
|
||||
//opa env vars
|
||||
pub const ENV_POLICY_PLUGIN_OPA_URL: &str = "RUSTFS_POLICY_PLUGIN_URL";
|
||||
pub const ENV_POLICY_PLUGIN_AUTH_TOKEN: &str = "RUSTFS_POLICY_PLUGIN_AUTH_TOKEN";
|
||||
|
||||
/// Start the log processing worker thread
|
||||
pub(crate) async fn start_worker(receiver: Receiver<UnifiedLogEntry>, sinks: Vec<Arc<dyn Sink>>) {
|
||||
let mut receiver = receiver;
|
||||
while let Some(entry) = receiver.recv().await {
|
||||
for sink in &sinks {
|
||||
sink.write(&entry).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
pub const ENV_POLICY_PLUGIN_KEYS: &[&str] = &[ENV_POLICY_PLUGIN_OPA_URL, ENV_POLICY_PLUGIN_AUTH_TOKEN];
|
||||
|
||||
pub const POLICY_PLUGIN_SUB_SYS: &str = "policy_plugin";
|
||||
@@ -49,4 +49,4 @@ uuid = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
md5 = { workspace = true }
|
||||
md5 = { workspace = true }
|
||||
@@ -20,8 +20,8 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::HeaderMap;
|
||||
use std::io::Cursor;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use s3s::dto::ETag;
|
||||
use std::{collections::HashMap, io::Cursor, sync::Arc};
|
||||
use tokio::io::BufReader;
|
||||
|
||||
use crate::error::ErrorResponse;
|
||||
@@ -148,27 +148,30 @@ pub fn new_getobjectreader(
|
||||
Ok((get_fn, off as i64, length as i64))
|
||||
}
|
||||
|
||||
/// Format an ETag value according to HTTP standards (wrap with quotes if not already wrapped)
|
||||
pub fn format_etag(etag: &str) -> String {
|
||||
if etag.starts_with('"') && etag.ends_with('"') {
|
||||
// Already properly formatted
|
||||
etag.to_string()
|
||||
} else if etag.starts_with("W/\"") && etag.ends_with('"') {
|
||||
// Already a weak ETag, properly formatted
|
||||
etag.to_string()
|
||||
} else {
|
||||
// Need to wrap with quotes
|
||||
format!("\"{}\"", etag)
|
||||
/// Convert a raw stored ETag into the strongly-typed `s3s::dto::ETag`.
|
||||
///
|
||||
/// Supports already quoted (`"abc"`), weak (`W/"abc"`), or plain (`abc`) values.
|
||||
pub fn to_s3s_etag(etag: &str) -> ETag {
|
||||
if let Some(rest) = etag.strip_prefix("W/\"") {
|
||||
if let Some(body) = rest.strip_suffix('"') {
|
||||
return ETag::Weak(body.to_string());
|
||||
}
|
||||
return ETag::Weak(rest.to_string());
|
||||
}
|
||||
|
||||
if let Some(body) = etag.strip_prefix('"').and_then(|rest| rest.strip_suffix('"')) {
|
||||
return ETag::Strong(body.to_string());
|
||||
}
|
||||
|
||||
ETag::Strong(etag.to_string())
|
||||
}
|
||||
|
||||
pub fn extract_etag(metadata: &HashMap<String, String>) -> String {
|
||||
let etag = if let Some(etag) = metadata.get("etag") {
|
||||
etag.clone()
|
||||
} else {
|
||||
metadata["md5Sum"].clone()
|
||||
};
|
||||
format_etag(&etag)
|
||||
pub fn get_raw_etag(metadata: &HashMap<String, String>) -> String {
|
||||
metadata
|
||||
.get("etag")
|
||||
.cloned()
|
||||
.or_else(|| metadata.get("md5Sum").cloned())
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -176,30 +179,28 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_format_etag() {
|
||||
// Test unquoted ETag - should add quotes
|
||||
assert_eq!(format_etag("6af8d12c0c74b78094884349f3c8a079"), "\"6af8d12c0c74b78094884349f3c8a079\"");
|
||||
|
||||
// Test already quoted ETag - should not double quote
|
||||
fn test_to_s3s_etag() {
|
||||
// Test unquoted ETag - should become strong etag
|
||||
assert_eq!(
|
||||
format_etag("\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
"\"6af8d12c0c74b78094884349f3c8a079\""
|
||||
to_s3s_etag("6af8d12c0c74b78094884349f3c8a079"),
|
||||
ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
);
|
||||
|
||||
// Test weak ETag - should keep as is
|
||||
assert_eq!(
|
||||
format_etag("W/\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
"W/\"6af8d12c0c74b78094884349f3c8a079\""
|
||||
to_s3s_etag("\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
);
|
||||
|
||||
// Test empty ETag - should add quotes
|
||||
assert_eq!(format_etag(""), "\"\"");
|
||||
assert_eq!(
|
||||
to_s3s_etag("W/\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
ETag::Weak("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
);
|
||||
|
||||
// Test malformed quote (only starting quote) - should wrap properly
|
||||
assert_eq!(format_etag("\"incomplete"), "\"\"incomplete\"");
|
||||
assert_eq!(to_s3s_etag(""), ETag::Strong(String::new()));
|
||||
|
||||
// Test malformed quote (only ending quote) - should wrap properly
|
||||
assert_eq!(format_etag("incomplete\""), "\"incomplete\"\"");
|
||||
assert_eq!(to_s3s_etag("\"incomplete"), ETag::Strong("\"incomplete".to_string()));
|
||||
|
||||
assert_eq!(to_s3s_etag("incomplete\""), ETag::Strong("incomplete\"".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -208,15 +209,17 @@ mod tests {
|
||||
|
||||
// Test with etag field
|
||||
metadata.insert("etag".to_string(), "abc123".to_string());
|
||||
assert_eq!(extract_etag(&metadata), "\"abc123\"");
|
||||
assert_eq!(get_raw_etag(&metadata), "abc123");
|
||||
|
||||
// Test with already quoted etag field
|
||||
metadata.insert("etag".to_string(), "\"def456\"".to_string());
|
||||
assert_eq!(extract_etag(&metadata), "\"def456\"");
|
||||
assert_eq!(get_raw_etag(&metadata), "\"def456\"");
|
||||
|
||||
// Test fallback to md5Sum
|
||||
metadata.remove("etag");
|
||||
metadata.insert("md5Sum".to_string(), "xyz789".to_string());
|
||||
assert_eq!(extract_etag(&metadata), "\"xyz789\"");
|
||||
assert_eq!(get_raw_etag(&metadata), "xyz789");
|
||||
|
||||
metadata.clear();
|
||||
assert_eq!(get_raw_etag(&metadata), "");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::replication::check_replicate_delete;
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::client::{object_api_utils::extract_etag, transition_api::ReaderImpl};
|
||||
use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
|
||||
use crate::disk::{
|
||||
@@ -4581,7 +4581,7 @@ impl StorageAPI for SetDisks {
|
||||
}*/
|
||||
// Normalize ETags by removing quotes before comparison (PR #592 compatibility)
|
||||
let transition_etag = rustfs_utils::path::trim_etag(&opts.transition.etag);
|
||||
let stored_etag = rustfs_utils::path::trim_etag(&extract_etag(&fi.metadata));
|
||||
let stored_etag = rustfs_utils::path::trim_etag(&get_raw_etag(&fi.metadata));
|
||||
if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp()
|
||||
|| transition_etag != stored_etag
|
||||
{
|
||||
|
||||
@@ -247,11 +247,16 @@ impl HTTPRangeSpec {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut start = 0i64;
|
||||
let mut end = -1i64;
|
||||
for i in 0..oi.parts.len().min(part_number) {
|
||||
if part_number == 0 || part_number > oi.parts.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut start = 0_i64;
|
||||
let mut end = -1_i64;
|
||||
for i in 0..part_number {
|
||||
let part = &oi.parts[i];
|
||||
start = end + 1;
|
||||
end = start + (oi.parts[i].size as i64) - 1
|
||||
end = start + (part.size as i64) - 1;
|
||||
}
|
||||
|
||||
Some(HTTPRangeSpec {
|
||||
@@ -266,8 +271,14 @@ impl HTTPRangeSpec {
|
||||
|
||||
let mut start = self.start;
|
||||
if self.is_suffix_length {
|
||||
start = res_size + self.start;
|
||||
|
||||
let suffix_len = if self.start < 0 {
|
||||
self.start
|
||||
.checked_neg()
|
||||
.ok_or_else(|| Error::other("range value invalid: suffix length overflow"))?
|
||||
} else {
|
||||
self.start
|
||||
};
|
||||
start = res_size - suffix_len;
|
||||
if start < 0 {
|
||||
start = 0;
|
||||
}
|
||||
@@ -280,7 +291,13 @@ impl HTTPRangeSpec {
|
||||
}
|
||||
|
||||
if self.is_suffix_length {
|
||||
let specified_len = self.start; // 假设 h.start 是一个 i64 类型
|
||||
let specified_len = if self.start < 0 {
|
||||
self.start
|
||||
.checked_neg()
|
||||
.ok_or_else(|| Error::other("range value invalid: suffix length overflow"))?
|
||||
} else {
|
||||
self.start
|
||||
};
|
||||
let mut range_length = specified_len;
|
||||
|
||||
if specified_len > res_size {
|
||||
@@ -460,7 +477,7 @@ impl From<s3s::dto::CompletedPart> for CompletePart {
|
||||
fn from(value: s3s::dto::CompletedPart) -> Self {
|
||||
Self {
|
||||
part_num: value.part_number.unwrap_or_default() as usize,
|
||||
etag: value.e_tag,
|
||||
etag: value.e_tag.map(|e| e.value().to_owned()),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1561,6 +1578,83 @@ mod tests {
|
||||
assert_eq!(length, 10); // end - start + 1 = 14 - 5 + 1 = 10
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_range_spec_suffix_positive_start() {
|
||||
let range_spec = HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: 5,
|
||||
end: -1,
|
||||
};
|
||||
|
||||
let (offset, length) = range_spec.get_offset_length(20).unwrap();
|
||||
assert_eq!(offset, 15);
|
||||
assert_eq!(length, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_range_spec_suffix_negative_start() {
|
||||
let range_spec = HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: -5,
|
||||
end: -1,
|
||||
};
|
||||
|
||||
let (offset, length) = range_spec.get_offset_length(20).unwrap();
|
||||
assert_eq!(offset, 15);
|
||||
assert_eq!(length, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_range_spec_suffix_exceeds_object() {
|
||||
let range_spec = HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: 50,
|
||||
end: -1,
|
||||
};
|
||||
|
||||
let (offset, length) = range_spec.get_offset_length(20).unwrap();
|
||||
assert_eq!(offset, 0);
|
||||
assert_eq!(length, 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_range_spec_from_object_info_valid_and_invalid_parts() {
|
||||
let object_info = ObjectInfo {
|
||||
size: 300,
|
||||
parts: vec![
|
||||
ObjectPartInfo {
|
||||
etag: String::new(),
|
||||
number: 1,
|
||||
size: 100,
|
||||
actual_size: 100,
|
||||
..Default::default()
|
||||
},
|
||||
ObjectPartInfo {
|
||||
etag: String::new(),
|
||||
number: 2,
|
||||
size: 100,
|
||||
actual_size: 100,
|
||||
..Default::default()
|
||||
},
|
||||
ObjectPartInfo {
|
||||
etag: String::new(),
|
||||
number: 3,
|
||||
size: 100,
|
||||
actual_size: 100,
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let spec = HTTPRangeSpec::from_object_info(&object_info, 2).unwrap();
|
||||
assert_eq!(spec.start, 100);
|
||||
assert_eq!(spec.end, 199);
|
||||
|
||||
assert!(HTTPRangeSpec::from_object_info(&object_info, 0).is_none());
|
||||
assert!(HTTPRangeSpec::from_object_info(&object_info, 4).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ranged_decompress_reader_zero_length() {
|
||||
let original_data = b"Hello, World!";
|
||||
|
||||
@@ -34,6 +34,7 @@ time = { workspace = true, features = ["serde-human-readable"] }
|
||||
serde = { workspace = true, features = ["derive", "rc"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-policy.workspace = true
|
||||
rustfs-config.workspace = true
|
||||
serde_json.workspace = true
|
||||
async-trait.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
@@ -35,14 +35,17 @@ use rustfs_policy::auth::{
|
||||
is_access_key_valid, is_secret_key_valid,
|
||||
};
|
||||
use rustfs_policy::policy::Args;
|
||||
use rustfs_policy::policy::opa;
|
||||
use rustfs_policy::policy::{EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, Policy, PolicyDoc, iam_policy_claim_name_sa};
|
||||
use rustfs_utils::crypto::{base64_decode, base64_encode};
|
||||
use serde_json::Value;
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
pub const MAX_SVCSESSION_POLICY_SIZE: usize = 4096;
|
||||
|
||||
@@ -53,6 +56,12 @@ pub const POLICYNAME: &str = "policy";
|
||||
pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
|
||||
pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
|
||||
|
||||
static POLICY_PLUGIN_CLIENT: OnceLock<Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>>> = OnceLock::new();
|
||||
|
||||
fn get_policy_plugin_client() -> Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>> {
|
||||
POLICY_PLUGIN_CLIENT.get_or_init(|| Arc::new(RwLock::new(None))).clone()
|
||||
}
|
||||
|
||||
pub struct IamSys<T> {
|
||||
store: Arc<IamCache<T>>,
|
||||
roles_map: HashMap<ARN, String>,
|
||||
@@ -60,6 +69,20 @@ pub struct IamSys<T> {
|
||||
|
||||
impl<T: Store> IamSys<T> {
|
||||
pub fn new(store: Arc<IamCache<T>>) -> Self {
|
||||
tokio::spawn(async move {
|
||||
match opa::lookup_config().await {
|
||||
Ok(conf) => {
|
||||
if conf.enable() {
|
||||
Self::set_policy_plugin_client(opa::AuthZPlugin::new(conf)).await;
|
||||
info!("OPA plugin enabled");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Error loading OPA configuration err:{}", e);
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
Self {
|
||||
store,
|
||||
roles_map: HashMap::new(),
|
||||
@@ -69,6 +92,18 @@ impl<T: Store> IamSys<T> {
|
||||
self.store.api.has_watcher()
|
||||
}
|
||||
|
||||
pub async fn set_policy_plugin_client(client: rustfs_policy::policy::opa::AuthZPlugin) {
|
||||
let policy_plugin_client = get_policy_plugin_client();
|
||||
let mut guard = policy_plugin_client.write().await;
|
||||
*guard = Some(client);
|
||||
}
|
||||
|
||||
pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
|
||||
let policy_plugin_client = get_policy_plugin_client();
|
||||
let guard = policy_plugin_client.read().await;
|
||||
guard.clone()
|
||||
}
|
||||
|
||||
pub async fn load_group(&self, name: &str) -> Result<()> {
|
||||
self.store.group_notification_handler(name).await
|
||||
}
|
||||
@@ -766,6 +801,11 @@ impl<T: Store> IamSys<T> {
|
||||
return true;
|
||||
}
|
||||
|
||||
let opa_enable = Self::get_policy_plugin_client().await;
|
||||
if let Some(opa_enable) = opa_enable {
|
||||
return opa_enable.is_allowed(args).await;
|
||||
}
|
||||
|
||||
let Ok((is_temp, parent_user)) = self.is_temp_user(args.account).await else { return false };
|
||||
|
||||
if is_temp {
|
||||
|
||||
@@ -635,7 +635,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
}
|
||||
|
||||
async fn encrypt(&self, request: EncryptRequest) -> Result<EncryptResponse> {
|
||||
let encrypt_request = crate::types::EncryptRequest {
|
||||
let encrypt_request = EncryptRequest {
|
||||
key_id: request.key_id.clone(),
|
||||
plaintext: request.plaintext,
|
||||
encryption_context: request.encryption_context,
|
||||
@@ -719,14 +719,14 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.client
|
||||
.load_master_key(key_id)
|
||||
.await
|
||||
.map_err(|_| crate::error::KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
.map_err(|_| KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
|
||||
let (deletion_date_str, deletion_date_dt) = if request.force_immediate.unwrap_or(false) {
|
||||
// For immediate deletion, actually delete the key from filesystem
|
||||
let key_path = self.client.master_key_path(key_id);
|
||||
tokio::fs::remove_file(&key_path)
|
||||
.await
|
||||
.map_err(|e| crate::error::KmsError::internal_error(format!("Failed to delete key file: {e}")))?;
|
||||
.map_err(|e| KmsError::internal_error(format!("Failed to delete key file: {e}")))?;
|
||||
|
||||
// Remove from cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
@@ -756,9 +756,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
// Schedule for deletion (default 30 days)
|
||||
let days = request.pending_window_in_days.unwrap_or(30);
|
||||
if !(7..=30).contains(&days) {
|
||||
return Err(crate::error::KmsError::invalid_parameter(
|
||||
"pending_window_in_days must be between 7 and 30".to_string(),
|
||||
));
|
||||
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30".to_string()));
|
||||
}
|
||||
|
||||
let deletion_date = chrono::Utc::now() + chrono::Duration::days(days as i64);
|
||||
@@ -772,16 +770,16 @@ impl KmsBackend for LocalKmsBackend {
|
||||
let key_path = self.client.master_key_path(key_id);
|
||||
let content = tokio::fs::read(&key_path)
|
||||
.await
|
||||
.map_err(|e| crate::error::KmsError::internal_error(format!("Failed to read key file: {e}")))?;
|
||||
let stored_key: crate::backends::local::StoredMasterKey = serde_json::from_slice(&content)
|
||||
.map_err(|e| crate::error::KmsError::internal_error(format!("Failed to parse stored key: {e}")))?;
|
||||
.map_err(|e| KmsError::internal_error(format!("Failed to read key file: {e}")))?;
|
||||
let stored_key: StoredMasterKey =
|
||||
serde_json::from_slice(&content).map_err(|e| KmsError::internal_error(format!("Failed to parse stored key: {e}")))?;
|
||||
|
||||
// Decrypt the existing key material to preserve it
|
||||
let existing_key_material = if let Some(ref cipher) = self.client.master_cipher {
|
||||
let nonce = aes_gcm::Nonce::from_slice(&stored_key.nonce);
|
||||
let nonce = Nonce::from_slice(&stored_key.nonce);
|
||||
cipher
|
||||
.decrypt(nonce, stored_key.encrypted_key_material.as_ref())
|
||||
.map_err(|e| crate::error::KmsError::cryptographic_error("decrypt", e.to_string()))?
|
||||
.map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string()))?
|
||||
} else {
|
||||
stored_key.encrypted_key_material
|
||||
};
|
||||
@@ -820,10 +818,10 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.client
|
||||
.load_master_key(key_id)
|
||||
.await
|
||||
.map_err(|_| crate::error::KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
.map_err(|_| KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
|
||||
if master_key.status != KeyStatus::PendingDeletion {
|
||||
return Err(crate::error::KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion")));
|
||||
return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion")));
|
||||
}
|
||||
|
||||
// Cancel the deletion by resetting the state
|
||||
|
||||
@@ -68,7 +68,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
key: WEBHOOK_QUEUE_DIR.to_string(),
|
||||
value: current_root
|
||||
.clone()
|
||||
.join("../../deploy/logs/notify/webhook")
|
||||
.join("../../deploy/logs/notify")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
@@ -120,11 +120,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
},
|
||||
KV {
|
||||
key: MQTT_QUEUE_DIR.to_string(),
|
||||
value: current_root
|
||||
.join("../../deploy/logs/notify/mqtt")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
value: current_root.join("../../deploy/logs/notify").to_str().unwrap().to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
@@ -137,7 +133,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
let mqtt_kvs = KVS(mqtt_kvs_vec);
|
||||
let mut mqtt_targets = std::collections::HashMap::new();
|
||||
mqtt_targets.insert(DEFAULT_TARGET.to_string(), mqtt_kvs);
|
||||
config.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_targets);
|
||||
// config.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_targets);
|
||||
|
||||
// Load the configuration and initialize the system
|
||||
*system.config.write().await = config;
|
||||
|
||||
@@ -28,6 +28,7 @@ use rustfs_targets::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use tracing::info;
|
||||
|
||||
#[tokio::main]
|
||||
@@ -68,7 +69,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
key: WEBHOOK_QUEUE_DIR.to_string(),
|
||||
value: current_root
|
||||
.clone()
|
||||
.join("../../deploy/logs/notify/webhook")
|
||||
.join("../../deploy/logs/notify")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
@@ -91,7 +92,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
system.init().await?;
|
||||
info!("✅ System initialized with Webhook target.");
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
|
||||
// --- Dynamically update system configuration: Add an MQTT Target ---
|
||||
info!("\n---> Dynamically adding MQTT target...");
|
||||
@@ -129,11 +130,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
},
|
||||
KV {
|
||||
key: MQTT_QUEUE_DIR.to_string(),
|
||||
value: current_root
|
||||
.join("../../deploy/logs/notify/mqtt")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
value: current_root.join("../../deploy/logs/notify").to_str().unwrap().to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
@@ -152,7 +149,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
.await?;
|
||||
info!("✅ MQTT target added and system reloaded.");
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
|
||||
// --- Loading and managing Bucket configurations ---
|
||||
info!("\n---> Loading bucket notification config...");
|
||||
@@ -176,7 +173,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
system.send_event(event).await;
|
||||
info!("✅ Event sent. Both Webhook and MQTT targets should receive it.");
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
|
||||
// --- Dynamically remove configuration ---
|
||||
info!("\n---> Dynamically removing Webhook target...");
|
||||
@@ -188,5 +185,6 @@ async fn main() -> Result<(), NotificationError> {
|
||||
info!("✅ Bucket 'my-bucket' config removed.");
|
||||
|
||||
info!("\nDemo completed successfully");
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -12,19 +12,20 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use axum::routing::get;
|
||||
use axum::{
|
||||
Router,
|
||||
extract::Json,
|
||||
extract::Query,
|
||||
http::{HeaderMap, Response, StatusCode},
|
||||
routing::post,
|
||||
routing::{get, post},
|
||||
};
|
||||
use rustfs_utils::parse_and_resolve_address;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
|
||||
use axum::extract::Query;
|
||||
use serde::Deserialize;
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ResetParams {
|
||||
@@ -32,9 +33,6 @@ struct ResetParams {
|
||||
}
|
||||
|
||||
// Define a global variable and count the number of data received
|
||||
use rustfs_utils::parse_and_resolve_address;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
static WEBHOOK_COUNT: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
|
||||
@@ -296,8 +296,8 @@ impl NotificationSystem {
|
||||
info!("Removing config for target {} of type {}", target_name, target_type);
|
||||
self.update_config_and_reload(|config| {
|
||||
let mut changed = false;
|
||||
if let Some(targets) = config.0.get_mut(target_type) {
|
||||
if targets.remove(target_name).is_some() {
|
||||
if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) {
|
||||
if targets.remove(&target_name.to_lowercase()).is_some() {
|
||||
changed = true;
|
||||
}
|
||||
if targets.is_empty() {
|
||||
@@ -307,6 +307,7 @@ impl NotificationSystem {
|
||||
if !changed {
|
||||
info!("Target {} of type {} not found, no changes made.", target_name, target_type);
|
||||
}
|
||||
debug!("Config after remove: {:?}", config);
|
||||
changed
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -16,12 +16,9 @@ use crate::Event;
|
||||
use crate::factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory};
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
use hashbrown::{HashMap, HashSet};
|
||||
use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
|
||||
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX};
|
||||
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, notify::NOTIFY_ROUTE_PREFIX};
|
||||
use rustfs_ecstore::config::{Config, KVS};
|
||||
use rustfs_targets::Target;
|
||||
use rustfs_targets::TargetError;
|
||||
use rustfs_targets::target::ChannelTargetType;
|
||||
use rustfs_targets::{Target, TargetError, target::ChannelTargetType};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
/// Registry for managing target factories
|
||||
@@ -90,7 +87,9 @@ impl TargetRegistry {
|
||||
let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
|
||||
// A collection of asynchronous tasks for concurrently executing target creation
|
||||
let mut tasks = FuturesUnordered::new();
|
||||
let mut final_config = config.clone(); // Clone a configuration for aggregating the final result
|
||||
// let final_config = config.clone(); // Clone a configuration for aggregating the final result
|
||||
// Record the defaults for each segment so that the segment can eventually be rebuilt
|
||||
let mut section_defaults: HashMap<String, KVS> = HashMap::new();
|
||||
// 1. Traverse all registered plants and process them by target type
|
||||
for (target_type, factory) in &self.factories {
|
||||
tracing::Span::current().record("target_type", target_type.as_str());
|
||||
@@ -98,12 +97,15 @@ impl TargetRegistry {
|
||||
|
||||
// 2. Prepare the configuration source
|
||||
// 2.1. Get the configuration segment in the file, e.g. 'notify_webhook'
|
||||
let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}");
|
||||
let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}").to_lowercase();
|
||||
let file_configs = config.0.get(§ion_name).cloned().unwrap_or_default();
|
||||
// 2.2. Get the default configuration for that type
|
||||
let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
|
||||
debug!(?default_cfg, "Get the default configuration");
|
||||
|
||||
// Save defaults for eventual write back
|
||||
section_defaults.insert(section_name.clone(), default_cfg.clone());
|
||||
|
||||
// *** Optimization point 1: Get all legitimate fields of the current target type ***
|
||||
let valid_fields = factory.get_valid_fields();
|
||||
debug!(?valid_fields, "Get the legitimate configuration fields");
|
||||
@@ -111,7 +113,9 @@ impl TargetRegistry {
|
||||
// 3. Resolve instance IDs and configuration overrides from environment variables
|
||||
let mut instance_ids_from_env = HashSet::new();
|
||||
// 3.1. Instance discovery: Based on the '..._ENABLE_INSTANCEID' format
|
||||
let enable_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}_{ENABLE_KEY}_").to_uppercase();
|
||||
let enable_prefix =
|
||||
format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
|
||||
.to_uppercase();
|
||||
for (key, value) in &all_env {
|
||||
if value.eq_ignore_ascii_case(rustfs_config::EnableState::One.as_str())
|
||||
|| value.eq_ignore_ascii_case(rustfs_config::EnableState::On.as_str())
|
||||
@@ -128,14 +132,14 @@ impl TargetRegistry {
|
||||
|
||||
// 3.2. Parse all relevant environment variable configurations
|
||||
// 3.2.1. Build environment variable prefixes such as 'RUSTFS_NOTIFY_WEBHOOK_'
|
||||
let env_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}_").to_uppercase();
|
||||
let env_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}").to_uppercase();
|
||||
// 3.2.2. 'env_overrides' is used to store configurations parsed from environment variables in the format: {instance id -> {field -> value}}
|
||||
let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();
|
||||
for (key, value) in &all_env {
|
||||
if let Some(rest) = key.strip_prefix(&env_prefix) {
|
||||
// Use rsplitn to split from the right side to properly extract the INSTANCE_ID at the end
|
||||
// Format: <FIELD_NAME>_<INSTANCE_ID> or <FIELD_NAME>
|
||||
let mut parts = rest.rsplitn(2, '_');
|
||||
let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
|
||||
|
||||
// The first part from the right is INSTANCE_ID
|
||||
let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
|
||||
@@ -224,7 +228,7 @@ impl TargetRegistry {
|
||||
} else {
|
||||
info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration");
|
||||
// Remove disabled target from final configuration
|
||||
final_config.0.entry(section_name.clone()).or_default().remove(&id);
|
||||
// final_config.0.entry(section_name.clone()).or_default().remove(&id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -246,15 +250,50 @@ impl TargetRegistry {
|
||||
}
|
||||
|
||||
// 7. Aggregate new configuration and write back to system configuration
|
||||
if !successful_configs.is_empty() {
|
||||
if !successful_configs.is_empty() || !section_defaults.is_empty() {
|
||||
info!(
|
||||
"Prepare to update {} successfully created target configurations to the system configuration...",
|
||||
successful_configs.len()
|
||||
);
|
||||
let mut new_config = config.clone();
|
||||
|
||||
let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
|
||||
|
||||
for (target_type, id, kvs) in successful_configs {
|
||||
let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}").to_lowercase();
|
||||
new_config.0.entry(section_name).or_default().insert(id, (*kvs).clone());
|
||||
successes_by_section
|
||||
.entry(section_name)
|
||||
.or_default()
|
||||
.insert(id.to_lowercase(), (*kvs).clone());
|
||||
}
|
||||
|
||||
let mut new_config = config.clone();
|
||||
// Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
|
||||
let mut sections: HashSet<String> = HashSet::new();
|
||||
sections.extend(section_defaults.keys().cloned());
|
||||
sections.extend(successes_by_section.keys().cloned());
|
||||
|
||||
for section in sections {
|
||||
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
|
||||
// Add default item
|
||||
if let Some(default_kvs) = section_defaults.get(§ion) {
|
||||
if !default_kvs.is_empty() {
|
||||
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Add successful instance item
|
||||
if let Some(instances) = successes_by_section.get(§ion) {
|
||||
for (id, kvs) in instances {
|
||||
section_map.insert(id.clone(), kvs.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Empty breaks are removed and non-empty breaks are replaced entirely.
|
||||
if section_map.is_empty() {
|
||||
new_config.0.remove(§ion);
|
||||
} else {
|
||||
new_config.0.insert(section, section_map);
|
||||
}
|
||||
}
|
||||
|
||||
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
|
||||
|
||||
@@ -29,17 +29,12 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/"
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = ["file"]
|
||||
file = []
|
||||
default = []
|
||||
gpu = ["dep:nvml-wrapper"]
|
||||
webhook = ["dep:reqwest"]
|
||||
kafka = ["dep:rdkafka"]
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["constants", "observability"] }
|
||||
rustfs-utils = { workspace = true, features = ["ip", "path"] }
|
||||
async-trait = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
flexi_logger = { workspace = true }
|
||||
nu-ansi-term = { workspace = true }
|
||||
nvml-wrapper = { workspace = true, optional = true }
|
||||
@@ -57,24 +52,9 @@ tracing-error = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
|
||||
reqwest = { workspace = true, optional = true }
|
||||
serde_json = { workspace = true }
|
||||
sysinfo = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
|
||||
# Only enable kafka features and related dependencies on Linux
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
rdkafka = { workspace = true, features = ["tokio"], optional = true }
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
chrono = { workspace = true }
|
||||
opentelemetry = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
opentelemetry-stdout = { workspace = true }
|
||||
opentelemetry-otlp = { workspace = true, features = ["grpc-tonic"] }
|
||||
opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
tracing = { workspace = true, features = ["std", "attributes"] }
|
||||
tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt"] }
|
||||
@@ -21,29 +21,4 @@ service_name = "rustfs"
|
||||
service_version = "0.1.0"
|
||||
environments = "develop"
|
||||
logger_level = "debug"
|
||||
local_logging_enabled = true # Default is false if not specified
|
||||
|
||||
|
||||
#[[sinks]]
|
||||
#type = "Kafka"
|
||||
#bootstrap_servers = "localhost:9092"
|
||||
#topic = "logs"
|
||||
#batch_size = 100 # Default is 100 if not specified
|
||||
#batch_timeout_ms = 100 # Default is 1000ms if not specified
|
||||
#
|
||||
#[[sinks]]
|
||||
#type = "Webhook"
|
||||
#endpoint = "http://localhost:8080/webhook"
|
||||
#auth_token = ""
|
||||
#batch_size = 100 # Default is 3 if not specified
|
||||
#batch_timeout_ms = 100 # Default is 100ms if not specified
|
||||
|
||||
[[sinks]]
|
||||
type = "File"
|
||||
path = "deploy/logs/rustfs.log"
|
||||
buffer_size = 102 # Default is 8192 bytes if not specified
|
||||
flush_interval_ms = 1000
|
||||
flush_threshold = 100
|
||||
|
||||
[logger]
|
||||
queue_capacity = 10000
|
||||
local_logging_enabled = true # Default is false if not specified
|
||||
@@ -13,33 +13,25 @@
|
||||
// limitations under the License.
|
||||
|
||||
use opentelemetry::global;
|
||||
use rustfs_obs::{BaseLogEntry, ServerLogEntry, SystemObserver, get_logger, init_obs, log_info};
|
||||
use std::collections::HashMap;
|
||||
use rustfs_obs::{SystemObserver, init_obs};
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tracing::{error, info, instrument};
|
||||
use tracing_core::Level;
|
||||
use tracing::{Level, error, info, instrument};
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let obs_conf = Some("crates/obs/examples/config.toml".to_string());
|
||||
let (_logger, _guard) = init_obs(obs_conf).await;
|
||||
let obs_conf = Some("http://localhost:4317".to_string());
|
||||
let _guard = init_obs(obs_conf).await;
|
||||
let span = tracing::span!(Level::INFO, "main");
|
||||
let _enter = span.enter();
|
||||
info!("Program starts");
|
||||
// Simulate the operation
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
run(
|
||||
"service-demo".to_string(),
|
||||
"object-demo".to_string(),
|
||||
"user-demo".to_string(),
|
||||
"service-demo".to_string(),
|
||||
)
|
||||
.await;
|
||||
run("service-demo".to_string()).await;
|
||||
info!("Program ends");
|
||||
}
|
||||
|
||||
#[instrument(fields(bucket, object, user))]
|
||||
async fn run(bucket: String, object: String, user: String, service_name: String) {
|
||||
async fn run(service_name: String) {
|
||||
let start_time = SystemTime::now();
|
||||
info!("Log module initialization is completed service_name: {:?}", service_name);
|
||||
|
||||
@@ -56,21 +48,6 @@ async fn run(bucket: String, object: String, user: String, service_name: String)
|
||||
Err(e) => error!("Failed to initialize process observer: {:?}", e),
|
||||
}
|
||||
|
||||
let base_entry = BaseLogEntry::new()
|
||||
.message(Some("run logger api_handler info".to_string()))
|
||||
.request_id(Some("request_id".to_string()))
|
||||
.timestamp(chrono::DateTime::from(start_time))
|
||||
.tags(Some(HashMap::default()));
|
||||
|
||||
let server_entry = ServerLogEntry::new(Level::INFO, "api_handler".to_string())
|
||||
.with_base(base_entry)
|
||||
.user_id(Some(user.clone()))
|
||||
.add_field("operation".to_string(), "login".to_string())
|
||||
.add_field("bucket".to_string(), bucket.clone())
|
||||
.add_field("object".to_string(), object.clone());
|
||||
|
||||
let result = get_logger().lock().await.log_server_entry(server_entry).await;
|
||||
info!("Logging is completed {:?}", result);
|
||||
put_object("bucket".to_string(), "object".to_string(), "user".to_string()).await;
|
||||
info!("Logging is completed");
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
@@ -97,8 +74,6 @@ async fn put_object(bucket: String, object: String, user: String) {
|
||||
start_time.elapsed().unwrap().as_secs_f64()
|
||||
);
|
||||
|
||||
let result = log_info("put_object logger info", "put_object").await;
|
||||
info!("put_object is completed {:?}", result);
|
||||
// Simulate the operation
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
|
||||
@@ -13,16 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_config::observability::{
|
||||
DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY, DEFAULT_SINKS_FILE_BUFFER_SIZE, DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS,
|
||||
DEFAULT_SINKS_FILE_FLUSH_THRESHOLD, DEFAULT_SINKS_KAFKA_BATCH_SIZE, DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS,
|
||||
DEFAULT_SINKS_KAFKA_BROKERS, DEFAULT_SINKS_KAFKA_TOPIC, DEFAULT_SINKS_WEBHOOK_AUTH_TOKEN, DEFAULT_SINKS_WEBHOOK_ENDPOINT,
|
||||
DEFAULT_SINKS_WEBHOOK_MAX_RETRIES, DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS, ENV_AUDIT_LOGGER_QUEUE_CAPACITY, ENV_OBS_ENDPOINT,
|
||||
ENV_OBS_ENVIRONMENT, ENV_OBS_LOCAL_LOGGING_ENABLED, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_KEEP_FILES,
|
||||
ENV_OBS_LOG_ROTATION_SIZE_MB, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOGGER_LEVEL, ENV_OBS_METER_INTERVAL, ENV_OBS_SAMPLE_RATIO,
|
||||
ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_USE_STDOUT, ENV_SINKS_FILE_BUFFER_SIZE,
|
||||
ENV_SINKS_FILE_FLUSH_INTERVAL_MS, ENV_SINKS_FILE_FLUSH_THRESHOLD, ENV_SINKS_FILE_PATH, ENV_SINKS_KAFKA_BATCH_SIZE,
|
||||
ENV_SINKS_KAFKA_BATCH_TIMEOUT_MS, ENV_SINKS_KAFKA_BROKERS, ENV_SINKS_KAFKA_TOPIC, ENV_SINKS_WEBHOOK_AUTH_TOKEN,
|
||||
ENV_SINKS_WEBHOOK_ENDPOINT, ENV_SINKS_WEBHOOK_MAX_RETRIES, ENV_SINKS_WEBHOOK_RETRY_DELAY_MS,
|
||||
ENV_OBS_ENDPOINT, ENV_OBS_ENVIRONMENT, ENV_OBS_LOCAL_LOGGING_ENABLED, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_FILENAME,
|
||||
ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_ROTATION_SIZE_MB, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOGGER_LEVEL,
|
||||
ENV_OBS_METER_INTERVAL, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_USE_STDOUT,
|
||||
};
|
||||
use rustfs_config::{
|
||||
APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_SIZE_MB, DEFAULT_LOG_ROTATION_TIME,
|
||||
@@ -145,167 +138,10 @@ impl Default for OtelConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Kafka Sink Configuration - Add batch parameters
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct KafkaSinkConfig {
|
||||
pub brokers: String,
|
||||
pub topic: String,
|
||||
pub batch_size: Option<usize>, // Batch size, default 100
|
||||
pub batch_timeout_ms: Option<u64>, // Batch timeout time, default 1000ms
|
||||
}
|
||||
|
||||
impl KafkaSinkConfig {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for KafkaSinkConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
brokers: env::var(ENV_SINKS_KAFKA_BROKERS)
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_SINKS_KAFKA_BROKERS.to_string()),
|
||||
topic: env::var(ENV_SINKS_KAFKA_TOPIC)
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_SINKS_KAFKA_TOPIC.to_string()),
|
||||
batch_size: env::var(ENV_SINKS_KAFKA_BATCH_SIZE)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_KAFKA_BATCH_SIZE)),
|
||||
batch_timeout_ms: env::var(ENV_SINKS_KAFKA_BATCH_TIMEOUT_MS)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Webhook Sink Configuration - Add Retry Parameters
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct WebhookSinkConfig {
|
||||
pub endpoint: String,
|
||||
pub auth_token: String,
|
||||
pub max_retries: Option<usize>, // Maximum number of retry times, default 3
|
||||
pub retry_delay_ms: Option<u64>, // Retry the delay cardinality, default 100ms
|
||||
}
|
||||
|
||||
impl WebhookSinkConfig {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for WebhookSinkConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
endpoint: env::var(ENV_SINKS_WEBHOOK_ENDPOINT)
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_SINKS_WEBHOOK_ENDPOINT.to_string()),
|
||||
auth_token: env::var(ENV_SINKS_WEBHOOK_AUTH_TOKEN)
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_SINKS_WEBHOOK_AUTH_TOKEN.to_string()),
|
||||
max_retries: env::var(ENV_SINKS_WEBHOOK_MAX_RETRIES)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_WEBHOOK_MAX_RETRIES)),
|
||||
retry_delay_ms: env::var(ENV_SINKS_WEBHOOK_RETRY_DELAY_MS)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// File Sink Configuration - Add buffering parameters
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct FileSinkConfig {
|
||||
pub path: String,
|
||||
pub buffer_size: Option<usize>, // Write buffer size, default 8192
|
||||
pub flush_interval_ms: Option<u64>, // Refresh interval time, default 1000ms
|
||||
pub flush_threshold: Option<usize>, // Refresh threshold, default 100 logs
|
||||
}
|
||||
|
||||
impl FileSinkConfig {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FileSinkConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
path: get_log_directory_to_string(ENV_SINKS_FILE_PATH),
|
||||
buffer_size: env::var(ENV_SINKS_FILE_BUFFER_SIZE)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_FILE_BUFFER_SIZE)),
|
||||
flush_interval_ms: env::var(ENV_SINKS_FILE_FLUSH_INTERVAL_MS)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS)),
|
||||
flush_threshold: env::var(ENV_SINKS_FILE_FLUSH_THRESHOLD)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_FILE_FLUSH_THRESHOLD)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sink configuration collection
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum SinkConfig {
|
||||
File(FileSinkConfig),
|
||||
Kafka(KafkaSinkConfig),
|
||||
Webhook(WebhookSinkConfig),
|
||||
}
|
||||
|
||||
impl SinkConfig {
|
||||
pub fn new() -> Self {
|
||||
Self::File(FileSinkConfig::new())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SinkConfig {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
///Logger Configuration
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct LoggerConfig {
|
||||
pub queue_capacity: Option<usize>,
|
||||
}
|
||||
|
||||
impl LoggerConfig {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
queue_capacity: env::var(ENV_AUDIT_LOGGER_QUEUE_CAPACITY)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LoggerConfig {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Overall application configuration
|
||||
/// Add observability, sinks, and logger configuration
|
||||
/// Add observability configuration
|
||||
///
|
||||
/// Observability: OpenTelemetry configuration
|
||||
/// Sinks: Kafka, Webhook, File sink configuration
|
||||
/// Logger: Logger configuration
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
@@ -316,8 +152,6 @@ impl Default for LoggerConfig {
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct AppConfig {
|
||||
pub observability: OtelConfig,
|
||||
pub sinks: Vec<SinkConfig>,
|
||||
pub logger: Option<LoggerConfig>,
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
@@ -328,16 +162,12 @@ impl AppConfig {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
observability: OtelConfig::default(),
|
||||
sinks: vec![SinkConfig::default()],
|
||||
logger: Some(LoggerConfig::default()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_with_endpoint(endpoint: Option<String>) -> Self {
|
||||
Self {
|
||||
observability: OtelConfig::extract_otel_config_from_env(endpoint),
|
||||
sinks: vec![SinkConfig::new()],
|
||||
logger: Some(LoggerConfig::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,88 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::entry::ObjectVersion;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Args - defines the arguments for API operations
|
||||
/// Args is used to define the arguments for API operations.
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::Args;
|
||||
/// use std::collections::HashMap;
|
||||
///
|
||||
/// let args = Args::new()
|
||||
/// .set_bucket(Some("my-bucket".to_string()))
|
||||
/// .set_object(Some("my-object".to_string()))
|
||||
/// .set_version_id(Some("123".to_string()))
|
||||
/// .set_metadata(Some(HashMap::new()));
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default, Eq, PartialEq)]
|
||||
pub struct Args {
|
||||
#[serde(rename = "bucket", skip_serializing_if = "Option::is_none")]
|
||||
pub bucket: Option<String>,
|
||||
#[serde(rename = "object", skip_serializing_if = "Option::is_none")]
|
||||
pub object: Option<String>,
|
||||
#[serde(rename = "versionId", skip_serializing_if = "Option::is_none")]
|
||||
pub version_id: Option<String>,
|
||||
#[serde(rename = "objects", skip_serializing_if = "Option::is_none")]
|
||||
pub objects: Option<Vec<ObjectVersion>>,
|
||||
#[serde(rename = "metadata", skip_serializing_if = "Option::is_none")]
|
||||
pub metadata: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
impl Args {
|
||||
/// Create a new Args object
|
||||
pub fn new() -> Self {
|
||||
Args {
|
||||
bucket: None,
|
||||
object: None,
|
||||
version_id: None,
|
||||
objects: None,
|
||||
metadata: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the bucket
|
||||
pub fn set_bucket(mut self, bucket: Option<String>) -> Self {
|
||||
self.bucket = bucket;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object
|
||||
pub fn set_object(mut self, object: Option<String>) -> Self {
|
||||
self.object = object;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the version ID
|
||||
pub fn set_version_id(mut self, version_id: Option<String>) -> Self {
|
||||
self.version_id = version_id;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the objects
|
||||
pub fn set_objects(mut self, objects: Option<Vec<ObjectVersion>>) -> Self {
|
||||
self.objects = objects;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the metadata
|
||||
pub fn set_metadata(mut self, metadata: Option<HashMap<String, String>>) -> Self {
|
||||
self.metadata = metadata;
|
||||
self
|
||||
}
|
||||
}
|
||||
@@ -1,467 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{BaseLogEntry, LogRecord, ObjectVersion};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// API details structure
|
||||
/// ApiDetails is used to define the details of an API operation
|
||||
///
|
||||
/// The `ApiDetails` structure contains the following fields:
|
||||
/// - `name` - the name of the API operation
|
||||
/// - `bucket` - the bucket name
|
||||
/// - `object` - the object name
|
||||
/// - `objects` - the list of objects
|
||||
/// - `status` - the status of the API operation
|
||||
/// - `status_code` - the status code of the API operation
|
||||
/// - `input_bytes` - the input bytes
|
||||
/// - `output_bytes` - the output bytes
|
||||
/// - `header_bytes` - the header bytes
|
||||
/// - `time_to_first_byte` - the time to first byte
|
||||
/// - `time_to_first_byte_in_ns` - the time to first byte in nanoseconds
|
||||
/// - `time_to_response` - the time to response
|
||||
/// - `time_to_response_in_ns` - the time to response in nanoseconds
|
||||
///
|
||||
/// The `ApiDetails` structure contains the following methods:
|
||||
/// - `new` - create a new `ApiDetails` with default values
|
||||
/// - `set_name` - set the name
|
||||
/// - `set_bucket` - set the bucket
|
||||
/// - `set_object` - set the object
|
||||
/// - `set_objects` - set the objects
|
||||
/// - `set_status` - set the status
|
||||
/// - `set_status_code` - set the status code
|
||||
/// - `set_input_bytes` - set the input bytes
|
||||
/// - `set_output_bytes` - set the output bytes
|
||||
/// - `set_header_bytes` - set the header bytes
|
||||
/// - `set_time_to_first_byte` - set the time to first byte
|
||||
/// - `set_time_to_first_byte_in_ns` - set the time to first byte in nanoseconds
|
||||
/// - `set_time_to_response` - set the time to response
|
||||
/// - `set_time_to_response_in_ns` - set the time to response in nanoseconds
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::ApiDetails;
|
||||
/// use rustfs_obs::ObjectVersion;
|
||||
///
|
||||
/// let api = ApiDetails::new()
|
||||
/// .set_name(Some("GET".to_string()))
|
||||
/// .set_bucket(Some("my-bucket".to_string()))
|
||||
/// .set_object(Some("my-object".to_string()))
|
||||
/// .set_objects(vec![ObjectVersion::new_with_object_name("my-object".to_string())])
|
||||
/// .set_status(Some("OK".to_string()))
|
||||
/// .set_status_code(Some(200))
|
||||
/// .set_input_bytes(100)
|
||||
/// .set_output_bytes(200)
|
||||
/// .set_header_bytes(Some(50))
|
||||
/// .set_time_to_first_byte(Some("100ms".to_string()))
|
||||
/// .set_time_to_first_byte_in_ns(Some("100000000ns".to_string()))
|
||||
/// .set_time_to_response(Some("200ms".to_string()))
|
||||
/// .set_time_to_response_in_ns(Some("200000000ns".to_string()));
|
||||
/// ```
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ApiDetails {
|
||||
#[serde(rename = "name", skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
#[serde(rename = "bucket", skip_serializing_if = "Option::is_none")]
|
||||
pub bucket: Option<String>,
|
||||
#[serde(rename = "object", skip_serializing_if = "Option::is_none")]
|
||||
pub object: Option<String>,
|
||||
#[serde(rename = "objects", skip_serializing_if = "Vec::is_empty", default)]
|
||||
pub objects: Vec<ObjectVersion>,
|
||||
#[serde(rename = "status", skip_serializing_if = "Option::is_none")]
|
||||
pub status: Option<String>,
|
||||
#[serde(rename = "statusCode", skip_serializing_if = "Option::is_none")]
|
||||
pub status_code: Option<i32>,
|
||||
#[serde(rename = "rx")]
|
||||
pub input_bytes: i64,
|
||||
#[serde(rename = "tx")]
|
||||
pub output_bytes: i64,
|
||||
#[serde(rename = "txHeaders", skip_serializing_if = "Option::is_none")]
|
||||
pub header_bytes: Option<i64>,
|
||||
#[serde(rename = "timeToFirstByte", skip_serializing_if = "Option::is_none")]
|
||||
pub time_to_first_byte: Option<String>,
|
||||
#[serde(rename = "timeToFirstByteInNS", skip_serializing_if = "Option::is_none")]
|
||||
pub time_to_first_byte_in_ns: Option<String>,
|
||||
#[serde(rename = "timeToResponse", skip_serializing_if = "Option::is_none")]
|
||||
pub time_to_response: Option<String>,
|
||||
#[serde(rename = "timeToResponseInNS", skip_serializing_if = "Option::is_none")]
|
||||
pub time_to_response_in_ns: Option<String>,
|
||||
}
|
||||
|
||||
impl ApiDetails {
|
||||
/// Create a new `ApiDetails` with default values
|
||||
pub fn new() -> Self {
|
||||
ApiDetails {
|
||||
name: None,
|
||||
bucket: None,
|
||||
object: None,
|
||||
objects: Vec::new(),
|
||||
status: None,
|
||||
status_code: None,
|
||||
input_bytes: 0,
|
||||
output_bytes: 0,
|
||||
header_bytes: None,
|
||||
time_to_first_byte: None,
|
||||
time_to_first_byte_in_ns: None,
|
||||
time_to_response: None,
|
||||
time_to_response_in_ns: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the name
|
||||
pub fn set_name(mut self, name: Option<String>) -> Self {
|
||||
self.name = name;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the bucket
|
||||
pub fn set_bucket(mut self, bucket: Option<String>) -> Self {
|
||||
self.bucket = bucket;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object
|
||||
pub fn set_object(mut self, object: Option<String>) -> Self {
|
||||
self.object = object;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the objects
|
||||
pub fn set_objects(mut self, objects: Vec<ObjectVersion>) -> Self {
|
||||
self.objects = objects;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the status
|
||||
pub fn set_status(mut self, status: Option<String>) -> Self {
|
||||
self.status = status;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the status code
|
||||
pub fn set_status_code(mut self, status_code: Option<i32>) -> Self {
|
||||
self.status_code = status_code;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the input bytes
|
||||
pub fn set_input_bytes(mut self, input_bytes: i64) -> Self {
|
||||
self.input_bytes = input_bytes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the output bytes
|
||||
pub fn set_output_bytes(mut self, output_bytes: i64) -> Self {
|
||||
self.output_bytes = output_bytes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the header bytes
|
||||
pub fn set_header_bytes(mut self, header_bytes: Option<i64>) -> Self {
|
||||
self.header_bytes = header_bytes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the time to first byte
|
||||
pub fn set_time_to_first_byte(mut self, time_to_first_byte: Option<String>) -> Self {
|
||||
self.time_to_first_byte = time_to_first_byte;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the time to first byte in nanoseconds
|
||||
pub fn set_time_to_first_byte_in_ns(mut self, time_to_first_byte_in_ns: Option<String>) -> Self {
|
||||
self.time_to_first_byte_in_ns = time_to_first_byte_in_ns;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the time to response
|
||||
pub fn set_time_to_response(mut self, time_to_response: Option<String>) -> Self {
|
||||
self.time_to_response = time_to_response;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the time to response in nanoseconds
|
||||
pub fn set_time_to_response_in_ns(mut self, time_to_response_in_ns: Option<String>) -> Self {
|
||||
self.time_to_response_in_ns = time_to_response_in_ns;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Entry - audit entry logs
|
||||
/// AuditLogEntry is used to define the structure of an audit log entry
|
||||
///
|
||||
/// The `AuditLogEntry` structure contains the following fields:
|
||||
/// - `base` - the base log entry
|
||||
/// - `version` - the version of the audit log entry
|
||||
/// - `deployment_id` - the deployment ID
|
||||
/// - `event` - the event
|
||||
/// - `entry_type` - the type of audit message
|
||||
/// - `api` - the API details
|
||||
/// - `remote_host` - the remote host
|
||||
/// - `user_agent` - the user agent
|
||||
/// - `req_path` - the request path
|
||||
/// - `req_host` - the request host
|
||||
/// - `req_claims` - the request claims
|
||||
/// - `req_query` - the request query
|
||||
/// - `req_header` - the request header
|
||||
/// - `resp_header` - the response header
|
||||
/// - `access_key` - the access key
|
||||
/// - `parent_user` - the parent user
|
||||
/// - `error` - the error
|
||||
///
|
||||
/// The `AuditLogEntry` structure contains the following methods:
|
||||
/// - `new` - create a new `AuditEntry` with default values
|
||||
/// - `new_with_values` - create a new `AuditEntry` with version, time, event and api details
|
||||
/// - `with_base` - set the base log entry
|
||||
/// - `set_version` - set the version
|
||||
/// - `set_deployment_id` - set the deployment ID
|
||||
/// - `set_event` - set the event
|
||||
/// - `set_entry_type` - set the entry type
|
||||
/// - `set_api` - set the API details
|
||||
/// - `set_remote_host` - set the remote host
|
||||
/// - `set_user_agent` - set the user agent
|
||||
/// - `set_req_path` - set the request path
|
||||
/// - `set_req_host` - set the request host
|
||||
/// - `set_req_claims` - set the request claims
|
||||
/// - `set_req_query` - set the request query
|
||||
/// - `set_req_header` - set the request header
|
||||
/// - `set_resp_header` - set the response header
|
||||
/// - `set_access_key` - set the access key
|
||||
/// - `set_parent_user` - set the parent user
|
||||
/// - `set_error` - set the error
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::AuditLogEntry;
|
||||
/// use rustfs_obs::ApiDetails;
|
||||
/// use std::collections::HashMap;
|
||||
///
|
||||
/// let entry = AuditLogEntry::new()
|
||||
/// .set_version("1.0".to_string())
|
||||
/// .set_deployment_id(Some("123".to_string()))
|
||||
/// .set_event("event".to_string())
|
||||
/// .set_entry_type(Some("type".to_string()))
|
||||
/// .set_api(ApiDetails::new())
|
||||
/// .set_remote_host(Some("remote-host".to_string()))
|
||||
/// .set_user_agent(Some("user-agent".to_string()))
|
||||
/// .set_req_path(Some("req-path".to_string()))
|
||||
/// .set_req_host(Some("req-host".to_string()))
|
||||
/// .set_req_claims(Some(HashMap::new()))
|
||||
/// .set_req_query(Some(HashMap::new()))
|
||||
/// .set_req_header(Some(HashMap::new()))
|
||||
/// .set_resp_header(Some(HashMap::new()))
|
||||
/// .set_access_key(Some("access-key".to_string()))
|
||||
/// .set_parent_user(Some("parent-user".to_string()))
|
||||
/// .set_error(Some("error".to_string()));
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, Default)]
|
||||
pub struct AuditLogEntry {
|
||||
#[serde(flatten)]
|
||||
pub base: BaseLogEntry,
|
||||
pub version: String,
|
||||
#[serde(rename = "deploymentid", skip_serializing_if = "Option::is_none")]
|
||||
pub deployment_id: Option<String>,
|
||||
pub event: String,
|
||||
// Class of audit message - S3, admin ops, bucket management
|
||||
#[serde(rename = "type", skip_serializing_if = "Option::is_none")]
|
||||
pub entry_type: Option<String>,
|
||||
pub api: ApiDetails,
|
||||
#[serde(rename = "remotehost", skip_serializing_if = "Option::is_none")]
|
||||
pub remote_host: Option<String>,
|
||||
#[serde(rename = "userAgent", skip_serializing_if = "Option::is_none")]
|
||||
pub user_agent: Option<String>,
|
||||
#[serde(rename = "requestPath", skip_serializing_if = "Option::is_none")]
|
||||
pub req_path: Option<String>,
|
||||
#[serde(rename = "requestHost", skip_serializing_if = "Option::is_none")]
|
||||
pub req_host: Option<String>,
|
||||
#[serde(rename = "requestClaims", skip_serializing_if = "Option::is_none")]
|
||||
pub req_claims: Option<HashMap<String, Value>>,
|
||||
#[serde(rename = "requestQuery", skip_serializing_if = "Option::is_none")]
|
||||
pub req_query: Option<HashMap<String, String>>,
|
||||
#[serde(rename = "requestHeader", skip_serializing_if = "Option::is_none")]
|
||||
pub req_header: Option<HashMap<String, String>>,
|
||||
#[serde(rename = "responseHeader", skip_serializing_if = "Option::is_none")]
|
||||
pub resp_header: Option<HashMap<String, String>>,
|
||||
#[serde(rename = "accessKey", skip_serializing_if = "Option::is_none")]
|
||||
pub access_key: Option<String>,
|
||||
#[serde(rename = "parentUser", skip_serializing_if = "Option::is_none")]
|
||||
pub parent_user: Option<String>,
|
||||
#[serde(rename = "error", skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
impl AuditLogEntry {
|
||||
/// Create a new `AuditEntry` with default values
|
||||
pub fn new() -> Self {
|
||||
AuditLogEntry {
|
||||
base: BaseLogEntry::new(),
|
||||
version: String::new(),
|
||||
deployment_id: None,
|
||||
event: String::new(),
|
||||
entry_type: None,
|
||||
api: ApiDetails::new(),
|
||||
remote_host: None,
|
||||
user_agent: None,
|
||||
req_path: None,
|
||||
req_host: None,
|
||||
req_claims: None,
|
||||
req_query: None,
|
||||
req_header: None,
|
||||
resp_header: None,
|
||||
access_key: None,
|
||||
parent_user: None,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new `AuditEntry` with version, time, event and api details
|
||||
pub fn new_with_values(version: String, time: DateTime<Utc>, event: String, api: ApiDetails) -> Self {
|
||||
let mut base = BaseLogEntry::new();
|
||||
base.timestamp = time;
|
||||
|
||||
AuditLogEntry {
|
||||
base,
|
||||
version,
|
||||
deployment_id: None,
|
||||
event,
|
||||
entry_type: None,
|
||||
api,
|
||||
remote_host: None,
|
||||
user_agent: None,
|
||||
req_path: None,
|
||||
req_host: None,
|
||||
req_claims: None,
|
||||
req_query: None,
|
||||
req_header: None,
|
||||
resp_header: None,
|
||||
access_key: None,
|
||||
parent_user: None,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the base log entry
|
||||
pub fn with_base(mut self, base: BaseLogEntry) -> Self {
|
||||
self.base = base;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the version
|
||||
pub fn set_version(mut self, version: String) -> Self {
|
||||
self.version = version;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the deployment ID
|
||||
pub fn set_deployment_id(mut self, deployment_id: Option<String>) -> Self {
|
||||
self.deployment_id = deployment_id;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the event
|
||||
pub fn set_event(mut self, event: String) -> Self {
|
||||
self.event = event;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the entry type
|
||||
pub fn set_entry_type(mut self, entry_type: Option<String>) -> Self {
|
||||
self.entry_type = entry_type;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the API details
|
||||
pub fn set_api(mut self, api: ApiDetails) -> Self {
|
||||
self.api = api;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the remote host
|
||||
pub fn set_remote_host(mut self, remote_host: Option<String>) -> Self {
|
||||
self.remote_host = remote_host;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the user agent
|
||||
pub fn set_user_agent(mut self, user_agent: Option<String>) -> Self {
|
||||
self.user_agent = user_agent;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request path
|
||||
pub fn set_req_path(mut self, req_path: Option<String>) -> Self {
|
||||
self.req_path = req_path;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request host
|
||||
pub fn set_req_host(mut self, req_host: Option<String>) -> Self {
|
||||
self.req_host = req_host;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request claims
|
||||
pub fn set_req_claims(mut self, req_claims: Option<HashMap<String, Value>>) -> Self {
|
||||
self.req_claims = req_claims;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request query
|
||||
pub fn set_req_query(mut self, req_query: Option<HashMap<String, String>>) -> Self {
|
||||
self.req_query = req_query;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request header
|
||||
pub fn set_req_header(mut self, req_header: Option<HashMap<String, String>>) -> Self {
|
||||
self.req_header = req_header;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the response header
|
||||
pub fn set_resp_header(mut self, resp_header: Option<HashMap<String, String>>) -> Self {
|
||||
self.resp_header = resp_header;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the access key
|
||||
pub fn set_access_key(mut self, access_key: Option<String>) -> Self {
|
||||
self.access_key = access_key;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the parent user
|
||||
pub fn set_parent_user(mut self, parent_user: Option<String>) -> Self {
|
||||
self.parent_user = parent_user;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the error
|
||||
pub fn set_error(mut self, error: Option<String>) -> Self {
|
||||
self.error = error;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl LogRecord for AuditLogEntry {
|
||||
fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_else(|_| String::from("{}"))
|
||||
}
|
||||
|
||||
fn get_timestamp(&self) -> DateTime<Utc> {
|
||||
self.base.timestamp
|
||||
}
|
||||
}
|
||||
@@ -1,106 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Base log entry structure shared by all log types
|
||||
/// This structure is used to serialize log entries to JSON
|
||||
/// and send them to the log sinks
|
||||
/// This structure is also used to deserialize log entries from JSON
|
||||
/// This structure is also used to store log entries in the database
|
||||
/// This structure is also used to query log entries from the database
|
||||
///
|
||||
/// The `BaseLogEntry` structure contains the following fields:
|
||||
/// - `timestamp` - the timestamp of the log entry
|
||||
/// - `request_id` - the request ID of the log entry
|
||||
/// - `message` - the message of the log entry
|
||||
/// - `tags` - the tags of the log entry
|
||||
///
|
||||
/// The `BaseLogEntry` structure contains the following methods:
|
||||
/// - `new` - create a new `BaseLogEntry` with default values
|
||||
/// - `message` - set the message
|
||||
/// - `request_id` - set the request ID
|
||||
/// - `tags` - set the tags
|
||||
/// - `timestamp` - set the timestamp
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::BaseLogEntry;
|
||||
/// use chrono::{DateTime, Utc};
|
||||
/// use std::collections::HashMap;
|
||||
///
|
||||
/// let timestamp = Utc::now();
|
||||
/// let request = Some("req-123".to_string());
|
||||
/// let message = Some("This is a log message".to_string());
|
||||
/// let tags = Some(HashMap::new());
|
||||
///
|
||||
/// let entry = BaseLogEntry::new()
|
||||
/// .timestamp(timestamp)
|
||||
/// .request_id(request)
|
||||
/// .message(message)
|
||||
/// .tags(tags);
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Default)]
|
||||
pub struct BaseLogEntry {
|
||||
#[serde(rename = "time")]
|
||||
pub timestamp: DateTime<Utc>,
|
||||
|
||||
#[serde(rename = "requestID", skip_serializing_if = "Option::is_none")]
|
||||
pub request_id: Option<String>,
|
||||
|
||||
#[serde(rename = "message", skip_serializing_if = "Option::is_none")]
|
||||
pub message: Option<String>,
|
||||
|
||||
#[serde(rename = "tags", skip_serializing_if = "Option::is_none")]
|
||||
pub tags: Option<HashMap<String, Value>>,
|
||||
}
|
||||
|
||||
impl BaseLogEntry {
|
||||
/// Create a new BaseLogEntry with default values
|
||||
pub fn new() -> Self {
|
||||
BaseLogEntry {
|
||||
timestamp: Utc::now(),
|
||||
request_id: None,
|
||||
message: None,
|
||||
tags: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the message
|
||||
pub fn message(mut self, message: Option<String>) -> Self {
|
||||
self.message = message;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request ID
|
||||
pub fn request_id(mut self, request_id: Option<String>) -> Self {
|
||||
self.request_id = request_id;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the tags
|
||||
pub fn tags(mut self, tags: Option<HashMap<String, Value>>) -> Self {
|
||||
self.tags = tags;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the timestamp
|
||||
pub fn timestamp(mut self, timestamp: DateTime<Utc>) -> Self {
|
||||
self.timestamp = timestamp;
|
||||
self
|
||||
}
|
||||
}
|
||||
@@ -1,158 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod args;
|
||||
pub(crate) mod audit;
|
||||
pub(crate) mod base;
|
||||
pub(crate) mod unified;
|
||||
|
||||
use serde::de::Error;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use tracing_core::Level;
|
||||
|
||||
/// ObjectVersion is used across multiple modules
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
|
||||
pub struct ObjectVersion {
|
||||
#[serde(rename = "name")]
|
||||
pub object_name: String,
|
||||
#[serde(rename = "versionId", skip_serializing_if = "Option::is_none")]
|
||||
pub version_id: Option<String>,
|
||||
}
|
||||
|
||||
impl ObjectVersion {
|
||||
/// Create a new ObjectVersion object
|
||||
pub fn new() -> Self {
|
||||
ObjectVersion {
|
||||
object_name: String::new(),
|
||||
version_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new ObjectVersion with object name
|
||||
pub fn new_with_object_name(object_name: String) -> Self {
|
||||
ObjectVersion {
|
||||
object_name,
|
||||
version_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the object name
|
||||
pub fn set_object_name(mut self, object_name: String) -> Self {
|
||||
self.object_name = object_name;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the version ID
|
||||
pub fn set_version_id(mut self, version_id: Option<String>) -> Self {
|
||||
self.version_id = version_id;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ObjectVersion {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Log kind/level enum
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
pub enum LogKind {
|
||||
#[serde(rename = "INFO")]
|
||||
#[default]
|
||||
Info,
|
||||
#[serde(rename = "WARNING")]
|
||||
Warning,
|
||||
#[serde(rename = "ERROR")]
|
||||
Error,
|
||||
#[serde(rename = "FATAL")]
|
||||
Fatal,
|
||||
}
|
||||
|
||||
/// Trait for types that can be serialized to JSON and have a timestamp
|
||||
/// This trait is used by `ServerLogEntry` to convert the log entry to JSON
|
||||
/// and get the timestamp of the log entry
|
||||
/// This trait is implemented by `ServerLogEntry`
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::LogRecord;
|
||||
/// use chrono::{DateTime, Utc};
|
||||
/// use rustfs_obs::ServerLogEntry;
|
||||
/// use tracing_core::Level;
|
||||
///
|
||||
/// let log_entry = ServerLogEntry::new(Level::INFO, "api_handler".to_string());
|
||||
/// let json = log_entry.to_json();
|
||||
/// let timestamp = log_entry.get_timestamp();
|
||||
/// ```
|
||||
pub trait LogRecord {
|
||||
fn to_json(&self) -> String;
|
||||
fn get_timestamp(&self) -> chrono::DateTime<chrono::Utc>;
|
||||
}
|
||||
|
||||
/// Wrapper for `tracing_core::Level` to implement `Serialize` and `Deserialize`
|
||||
/// for `ServerLogEntry`
|
||||
/// This is necessary because `tracing_core::Level` does not implement `Serialize`
|
||||
/// and `Deserialize`
|
||||
/// This is a workaround to allow `ServerLogEntry` to be serialized and deserialized
|
||||
/// using `serde`
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::SerializableLevel;
|
||||
/// use tracing_core::Level;
|
||||
///
|
||||
/// let level = Level::INFO;
|
||||
/// let serializable_level = SerializableLevel::from(level);
|
||||
/// ```
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SerializableLevel(pub Level);
|
||||
|
||||
impl From<Level> for SerializableLevel {
|
||||
fn from(level: Level) -> Self {
|
||||
SerializableLevel(level)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SerializableLevel> for Level {
|
||||
fn from(serializable_level: SerializableLevel) -> Self {
|
||||
serializable_level.0
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for SerializableLevel {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str(self.0.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for SerializableLevel {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let s = String::deserialize(deserializer)?;
|
||||
match s.as_str() {
|
||||
"TRACE" => Ok(SerializableLevel(Level::TRACE)),
|
||||
"DEBUG" => Ok(SerializableLevel(Level::DEBUG)),
|
||||
"INFO" => Ok(SerializableLevel(Level::INFO)),
|
||||
"WARN" => Ok(SerializableLevel(Level::WARN)),
|
||||
"ERROR" => Ok(SerializableLevel(Level::ERROR)),
|
||||
_ => Err(D::Error::custom("unknown log level")),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,301 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{AuditLogEntry, BaseLogEntry, LogKind, LogRecord, SerializableLevel};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing_core::Level;
|
||||
|
||||
/// Server log entry with structured fields
|
||||
/// ServerLogEntry is used to log structured log entries from the server
|
||||
///
|
||||
/// The `ServerLogEntry` structure contains the following fields:
|
||||
/// - `base` - the base log entry
|
||||
/// - `level` - the log level
|
||||
/// - `source` - the source of the log entry
|
||||
/// - `user_id` - the user ID
|
||||
/// - `fields` - the structured fields of the log entry
|
||||
///
|
||||
/// The `ServerLogEntry` structure contains the following methods:
|
||||
/// - `new` - create a new `ServerLogEntry` with specified level and source
|
||||
/// - `with_base` - set the base log entry
|
||||
/// - `user_id` - set the user ID
|
||||
/// - `fields` - set the fields
|
||||
/// - `add_field` - add a field
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::ServerLogEntry;
|
||||
/// use tracing_core::Level;
|
||||
///
|
||||
/// let entry = ServerLogEntry::new(Level::INFO, "test_module".to_string())
|
||||
/// .user_id(Some("user-456".to_string()))
|
||||
/// .add_field("operation".to_string(), "login".to_string());
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ServerLogEntry {
|
||||
#[serde(flatten)]
|
||||
pub base: BaseLogEntry,
|
||||
|
||||
pub level: SerializableLevel,
|
||||
pub source: String,
|
||||
|
||||
#[serde(rename = "userId", skip_serializing_if = "Option::is_none")]
|
||||
pub user_id: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Vec::is_empty", default)]
|
||||
pub fields: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl ServerLogEntry {
|
||||
/// Create a new ServerLogEntry with specified level and source
|
||||
pub fn new(level: Level, source: String) -> Self {
|
||||
ServerLogEntry {
|
||||
base: BaseLogEntry::new(),
|
||||
level: SerializableLevel(level),
|
||||
source,
|
||||
user_id: None,
|
||||
fields: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the base log entry
|
||||
pub fn with_base(mut self, base: BaseLogEntry) -> Self {
|
||||
self.base = base;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the user ID
|
||||
pub fn user_id(mut self, user_id: Option<String>) -> Self {
|
||||
self.user_id = user_id;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set fields
|
||||
pub fn fields(mut self, fields: Vec<(String, String)>) -> Self {
|
||||
self.fields = fields;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a field
|
||||
pub fn add_field(mut self, key: String, value: String) -> Self {
|
||||
self.fields.push((key, value));
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl LogRecord for ServerLogEntry {
|
||||
fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_else(|_| String::from("{}"))
|
||||
}
|
||||
|
||||
fn get_timestamp(&self) -> DateTime<Utc> {
|
||||
self.base.timestamp
|
||||
}
|
||||
}
|
||||
|
||||
/// Console log entry structure
|
||||
/// ConsoleLogEntry is used to log console log entries
|
||||
/// The `ConsoleLogEntry` structure contains the following fields:
|
||||
/// - `base` - the base log entry
|
||||
/// - `level` - the log level
|
||||
/// - `console_msg` - the console message
|
||||
/// - `node_name` - the node name
|
||||
/// - `err` - the error message
|
||||
///
|
||||
/// The `ConsoleLogEntry` structure contains the following methods:
|
||||
/// - `new` - create a new `ConsoleLogEntry`
|
||||
/// - `new_with_console_msg` - create a new `ConsoleLogEntry` with console message and node name
|
||||
/// - `with_base` - set the base log entry
|
||||
/// - `set_level` - set the log level
|
||||
/// - `set_node_name` - set the node name
|
||||
/// - `set_console_msg` - set the console message
|
||||
/// - `set_err` - set the error message
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::ConsoleLogEntry;
|
||||
///
|
||||
/// let entry = ConsoleLogEntry::new_with_console_msg("Test message".to_string(), "node-123".to_string());
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ConsoleLogEntry {
|
||||
#[serde(flatten)]
|
||||
pub base: BaseLogEntry,
|
||||
|
||||
pub level: LogKind,
|
||||
pub console_msg: String,
|
||||
pub node_name: String,
|
||||
|
||||
#[serde(skip)]
|
||||
pub err: Option<String>,
|
||||
}
|
||||
|
||||
impl ConsoleLogEntry {
|
||||
/// Create a new ConsoleLogEntry
|
||||
pub fn new() -> Self {
|
||||
ConsoleLogEntry {
|
||||
base: BaseLogEntry::new(),
|
||||
level: LogKind::Info,
|
||||
console_msg: String::new(),
|
||||
node_name: String::new(),
|
||||
err: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new ConsoleLogEntry with console message and node name
|
||||
pub fn new_with_console_msg(console_msg: String, node_name: String) -> Self {
|
||||
ConsoleLogEntry {
|
||||
base: BaseLogEntry::new(),
|
||||
level: LogKind::Info,
|
||||
console_msg,
|
||||
node_name,
|
||||
err: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the base log entry
|
||||
pub fn with_base(mut self, base: BaseLogEntry) -> Self {
|
||||
self.base = base;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the log level
|
||||
pub fn set_level(mut self, level: LogKind) -> Self {
|
||||
self.level = level;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the node name
|
||||
pub fn set_node_name(mut self, node_name: String) -> Self {
|
||||
self.node_name = node_name;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the console message
|
||||
pub fn set_console_msg(mut self, console_msg: String) -> Self {
|
||||
self.console_msg = console_msg;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the error message
|
||||
pub fn set_err(mut self, err: Option<String>) -> Self {
|
||||
self.err = err;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ConsoleLogEntry {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl LogRecord for ConsoleLogEntry {
|
||||
fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_else(|_| String::from("{}"))
|
||||
}
|
||||
|
||||
fn get_timestamp(&self) -> DateTime<Utc> {
|
||||
self.base.timestamp
|
||||
}
|
||||
}
|
||||
|
||||
/// Unified log entry type
|
||||
/// UnifiedLogEntry is used to log different types of log entries
|
||||
///
|
||||
/// The `UnifiedLogEntry` enum contains the following variants:
|
||||
/// - `Server` - a server log entry
|
||||
/// - `Audit` - an audit log entry
|
||||
/// - `Console` - a console log entry
|
||||
///
|
||||
/// The `UnifiedLogEntry` enum contains the following methods:
|
||||
/// - `to_json` - convert the log entry to JSON
|
||||
/// - `get_timestamp` - get the timestamp of the log entry
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::{UnifiedLogEntry, ServerLogEntry};
|
||||
/// use tracing_core::Level;
|
||||
///
|
||||
/// let server_entry = ServerLogEntry::new(Level::INFO, "test_module".to_string());
|
||||
/// let unified = UnifiedLogEntry::Server(server_entry);
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum UnifiedLogEntry {
|
||||
#[serde(rename = "server")]
|
||||
Server(ServerLogEntry),
|
||||
|
||||
#[serde(rename = "audit")]
|
||||
Audit(Box<AuditLogEntry>),
|
||||
|
||||
#[serde(rename = "console")]
|
||||
Console(ConsoleLogEntry),
|
||||
}
|
||||
|
||||
impl LogRecord for UnifiedLogEntry {
|
||||
fn to_json(&self) -> String {
|
||||
match self {
|
||||
UnifiedLogEntry::Server(entry) => entry.to_json(),
|
||||
UnifiedLogEntry::Audit(entry) => entry.to_json(),
|
||||
UnifiedLogEntry::Console(entry) => entry.to_json(),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_timestamp(&self) -> DateTime<Utc> {
|
||||
match self {
|
||||
UnifiedLogEntry::Server(entry) => entry.get_timestamp(),
|
||||
UnifiedLogEntry::Audit(entry) => entry.get_timestamp(),
|
||||
UnifiedLogEntry::Console(entry) => entry.get_timestamp(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_base_log_entry() {
|
||||
let base = BaseLogEntry::new()
|
||||
.request_id(Some("req-123".to_string()))
|
||||
.message(Some("Test message".to_string()));
|
||||
|
||||
assert_eq!(base.request_id, Some("req-123".to_string()));
|
||||
assert_eq!(base.message, Some("Test message".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_server_log_entry() {
|
||||
let entry = ServerLogEntry::new(Level::INFO, "test_module".to_string())
|
||||
.user_id(Some("user-456".to_string()))
|
||||
.add_field("operation".to_string(), "login".to_string());
|
||||
|
||||
assert_eq!(entry.level.0, Level::INFO);
|
||||
assert_eq!(entry.source, "test_module");
|
||||
assert_eq!(entry.user_id, Some("user-456".to_string()));
|
||||
assert_eq!(entry.fields.len(), 1);
|
||||
assert_eq!(entry.fields[0], ("operation".to_string(), "login".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unified_log_entry_json() {
|
||||
let server_entry = ServerLogEntry::new(Level::INFO, "test_source".to_string());
|
||||
let unified = UnifiedLogEntry::Server(server_entry);
|
||||
|
||||
let json = unified.to_json();
|
||||
assert!(json.contains("test_source"));
|
||||
}
|
||||
}
|
||||
@@ -12,9 +12,8 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::logger::InitLogStatus;
|
||||
use crate::AppConfig;
|
||||
use crate::telemetry::{OtelGuard, init_telemetry};
|
||||
use crate::{AppConfig, Logger, get_global_logger, init_global_logger};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::sync::{OnceCell, SetError};
|
||||
use tracing::{error, info};
|
||||
@@ -61,46 +60,14 @@ pub enum GlobalError {
|
||||
///
|
||||
/// # #[tokio::main]
|
||||
/// # async fn main() {
|
||||
/// let (logger, guard) = init_obs(None).await;
|
||||
/// let guard = init_obs(None).await;
|
||||
/// # }
|
||||
/// ```
|
||||
pub async fn init_obs(endpoint: Option<String>) -> (Arc<tokio::sync::Mutex<Logger>>, OtelGuard) {
|
||||
pub async fn init_obs(endpoint: Option<String>) -> OtelGuard {
|
||||
// Load the configuration file
|
||||
let config = AppConfig::new_with_endpoint(endpoint);
|
||||
|
||||
let guard = init_telemetry(&config.observability);
|
||||
|
||||
let logger = init_global_logger(&config).await;
|
||||
let obs_config = config.observability.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = InitLogStatus::init_start_log(&obs_config).await;
|
||||
match result {
|
||||
Ok(_) => {
|
||||
info!("Logger initialized successfully");
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to initialize logger: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
(logger, guard)
|
||||
}
|
||||
|
||||
/// Get the global logger instance
|
||||
/// This function returns a reference to the global logger instance.
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global logger instance
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::get_logger;
|
||||
///
|
||||
/// let logger = get_logger();
|
||||
/// ```
|
||||
pub fn get_logger() -> &'static Arc<tokio::sync::Mutex<Logger>> {
|
||||
get_global_logger()
|
||||
init_telemetry(&config.observability)
|
||||
}
|
||||
|
||||
/// Set the global guard for OpenTelemetry
|
||||
@@ -117,7 +84,7 @@ pub fn get_logger() -> &'static Arc<tokio::sync::Mutex<Logger>> {
|
||||
/// use rustfs_obs::{ init_obs, set_global_guard};
|
||||
///
|
||||
/// async fn init() -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let (_, guard) = init_obs(None).await;
|
||||
/// let guard = init_obs(None).await;
|
||||
/// set_global_guard(guard)?;
|
||||
/// Ok(())
|
||||
/// }
|
||||
|
||||
@@ -18,10 +18,7 @@
|
||||
//!
|
||||
//! ## feature mark
|
||||
//!
|
||||
//! - `file`: enable file logging enabled by default
|
||||
//! - `gpu`: gpu monitoring function
|
||||
//! - `kafka`: enable kafka metric output
|
||||
//! - `webhook`: enable webhook notifications
|
||||
//! - `full`: includes all functions
|
||||
//!
|
||||
//! to enable gpu monitoring add in cargo toml
|
||||
@@ -41,27 +38,15 @@
|
||||
///
|
||||
/// # #[tokio::main]
|
||||
/// # async fn main() {
|
||||
/// let (logger, guard) = init_obs(None).await;
|
||||
/// # let guard = init_obs(None).await;
|
||||
/// # }
|
||||
/// ```
|
||||
mod config;
|
||||
mod entry;
|
||||
mod global;
|
||||
mod logger;
|
||||
mod metrics;
|
||||
mod sinks;
|
||||
mod system;
|
||||
mod telemetry;
|
||||
mod worker;
|
||||
|
||||
pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig};
|
||||
pub use entry::args::Args;
|
||||
pub use entry::audit::{ApiDetails, AuditLogEntry};
|
||||
pub use entry::base::BaseLogEntry;
|
||||
pub use entry::unified::{ConsoleLogEntry, ServerLogEntry, UnifiedLogEntry};
|
||||
pub use entry::{LogKind, LogRecord, ObjectVersion, SerializableLevel};
|
||||
pub use config::AppConfig;
|
||||
pub use global::*;
|
||||
pub use logger::Logger;
|
||||
pub use logger::{get_global_logger, init_global_logger, start_logger};
|
||||
pub use logger::{log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
|
||||
pub use system::SystemObserver;
|
||||
|
||||
@@ -1,490 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::sinks::Sink;
|
||||
use crate::{
|
||||
AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry, sinks,
|
||||
};
|
||||
use rustfs_config::{APP_NAME, ENVIRONMENT, SERVICE_VERSION};
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||
use tokio::sync::{Mutex, OnceCell};
|
||||
use tracing_core::Level;
|
||||
|
||||
// Add the global instance at the module level
|
||||
static GLOBAL_LOGGER: OnceCell<Arc<Mutex<Logger>>> = OnceCell::const_new();
|
||||
|
||||
/// Server log processor
|
||||
#[derive(Debug)]
|
||||
pub struct Logger {
|
||||
sender: Sender<UnifiedLogEntry>, // Log sending channel
|
||||
queue_capacity: usize,
|
||||
}
|
||||
|
||||
impl Logger {
|
||||
/// Create a new Logger instance
|
||||
/// Returns Logger and corresponding Receiver
|
||||
pub fn new(config: &AppConfig) -> (Self, Receiver<UnifiedLogEntry>) {
|
||||
// Get queue capacity from configuration, or use default values 10000
|
||||
let queue_capacity = config.logger.as_ref().and_then(|l| l.queue_capacity).unwrap_or(10000);
|
||||
let (sender, receiver) = mpsc::channel(queue_capacity);
|
||||
(Logger { sender, queue_capacity }, receiver)
|
||||
}
|
||||
|
||||
/// get the queue capacity
|
||||
/// This function returns the queue capacity.
|
||||
/// # Returns
|
||||
/// The queue capacity
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::Logger;
|
||||
/// async fn example(logger: &Logger) {
|
||||
/// let _ = logger.get_queue_capacity();
|
||||
/// }
|
||||
/// ```
|
||||
pub fn get_queue_capacity(&self) -> usize {
|
||||
self.queue_capacity
|
||||
}
|
||||
|
||||
/// Log a server entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_server"))]
|
||||
pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Server(entry)).await
|
||||
}
|
||||
|
||||
/// Log an audit entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_audit"))]
|
||||
pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Audit(Box::new(entry))).await
|
||||
}
|
||||
|
||||
/// Log a console entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_console"))]
|
||||
pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Console(entry)).await
|
||||
}
|
||||
|
||||
/// Asynchronous logging of unified log entries
|
||||
#[tracing::instrument(skip_all, fields(log_source = "logger"))]
|
||||
pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), GlobalError> {
|
||||
// Extract information for tracing based on entry type
|
||||
match &entry {
|
||||
UnifiedLogEntry::Server(server) => {
|
||||
tracing::Span::current()
|
||||
.record("log_level", server.level.0.as_str())
|
||||
.record("log_message", server.base.message.as_deref().unwrap_or("log message not set"))
|
||||
.record("source", &server.source);
|
||||
|
||||
// Generate tracing event based on log level
|
||||
match server.level.0 {
|
||||
Level::ERROR => {
|
||||
tracing::error!(target: "server_logs", message = %server.base.message.as_deref().unwrap_or(""));
|
||||
}
|
||||
Level::WARN => {
|
||||
tracing::warn!(target: "server_logs", message = %server.base.message.as_deref().unwrap_or(""));
|
||||
}
|
||||
Level::INFO => {
|
||||
tracing::info!(target: "server_logs", message = %server.base.message.as_deref().unwrap_or(""));
|
||||
}
|
||||
Level::DEBUG => {
|
||||
tracing::debug!(target: "server_logs", message = %server.base.message.as_deref().unwrap_or(""));
|
||||
}
|
||||
Level::TRACE => {
|
||||
tracing::trace!(target: "server_logs", message = %server.base.message.as_deref().unwrap_or(""));
|
||||
}
|
||||
}
|
||||
}
|
||||
UnifiedLogEntry::Audit(audit) => {
|
||||
tracing::info!(
|
||||
target: "audit_logs",
|
||||
event = %audit.event,
|
||||
api = %audit.api.name.as_deref().unwrap_or("unknown"),
|
||||
message = %audit.base.message.as_deref().unwrap_or("")
|
||||
);
|
||||
}
|
||||
UnifiedLogEntry::Console(console) => {
|
||||
let level_str = match console.level {
|
||||
crate::LogKind::Info => "INFO",
|
||||
crate::LogKind::Warning => "WARN",
|
||||
crate::LogKind::Error => "ERROR",
|
||||
crate::LogKind::Fatal => "FATAL",
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
target: "console_logs",
|
||||
level = %level_str,
|
||||
node = %console.node_name,
|
||||
message = %console.console_msg
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Send logs to async queue with improved error handling
|
||||
match self.sender.try_send(entry) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(mpsc::error::TrySendError::Full(entry)) => {
|
||||
// Processing strategy when queue is full
|
||||
tracing::warn!("Log queue full, applying backpressure");
|
||||
match tokio::time::timeout(std::time::Duration::from_millis(500), self.sender.send(entry)).await {
|
||||
Ok(Ok(_)) => Ok(()),
|
||||
Ok(Err(_)) => Err(GlobalError::SendFailed("Channel closed")),
|
||||
Err(_) => Err(GlobalError::Timeout("Queue backpressure timeout")),
|
||||
}
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => Err(GlobalError::SendFailed("Logger channel closed")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Write log with context information
|
||||
/// This function writes log messages with context information.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// - `request_id`: Request ID
|
||||
/// - `user_id`: User ID
|
||||
/// - `fields`: Additional fields
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use tracing_core::Level;
|
||||
/// use rustfs_obs::Logger;
|
||||
///
|
||||
/// async fn example(logger: &Logger) {
|
||||
/// let _ = logger.write_with_context("This is an information message", "example",Level::INFO, Some("req-12345".to_string()), Some("user-6789".to_string()), vec![("endpoint".to_string(), "/api/v1/data".to_string())]).await;
|
||||
/// }
|
||||
pub async fn write_with_context(
|
||||
&self,
|
||||
message: &str,
|
||||
source: &str,
|
||||
level: Level,
|
||||
request_id: Option<String>,
|
||||
user_id: Option<String>,
|
||||
fields: Vec<(String, String)>,
|
||||
) -> Result<(), GlobalError> {
|
||||
let base = BaseLogEntry::new().message(Some(message.to_string())).request_id(request_id);
|
||||
|
||||
let server_entry = ServerLogEntry::new(level, source.to_string())
|
||||
.user_id(user_id)
|
||||
.fields(fields)
|
||||
.with_base(base);
|
||||
|
||||
self.log_server_entry(server_entry).await
|
||||
}
|
||||
|
||||
/// Write log
|
||||
/// This function writes log messages.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// - `level`: Log level
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::Logger;
|
||||
/// use tracing_core::Level;
|
||||
///
|
||||
/// async fn example(logger: &Logger) {
|
||||
/// let _ = logger.write("This is an information message", "example", Level::INFO).await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), GlobalError> {
|
||||
self.write_with_context(message, source, level, None, None, Vec::new()).await
|
||||
}
|
||||
|
||||
/// Shutdown the logger
|
||||
/// This function shuts down the logger.
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::Logger;
|
||||
///
|
||||
/// async fn example(logger: Logger) {
|
||||
/// let _ = logger.shutdown().await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn shutdown(self) -> Result<(), GlobalError> {
|
||||
drop(self.sender); //Close the sending end so that the receiver knows that there is no new message
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the log module
|
||||
/// This function starts the log module.
|
||||
/// It initializes the logger and starts the worker to process logs.
|
||||
/// # Parameters
|
||||
/// - `config`: Configuration information
|
||||
/// - `sinks`: A vector of Sink instances
|
||||
/// # Returns
|
||||
/// The global logger instance
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::{AppConfig, start_logger};
|
||||
///
|
||||
/// let config = AppConfig::default();
|
||||
/// let sinks = vec![];
|
||||
/// let logger = start_logger(&config, sinks);
|
||||
/// ```
|
||||
pub fn start_logger(config: &AppConfig, sinks: Vec<Arc<dyn Sink>>) -> Logger {
|
||||
let (logger, receiver) = Logger::new(config);
|
||||
tokio::spawn(crate::worker::start_worker(receiver, sinks));
|
||||
logger
|
||||
}
|
||||
|
||||
/// Initialize the global logger instance
|
||||
/// This function initializes the global logger instance and returns a reference to it.
|
||||
/// If the logger has been initialized before, it will return the existing logger instance.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `config`: Configuration information
|
||||
/// - `sinks`: A vector of Sink instances
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global logger instance
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::{AppConfig,init_global_logger};
|
||||
///
|
||||
/// let config = AppConfig::default();
|
||||
/// let logger = init_global_logger(&config);
|
||||
/// ```
|
||||
pub async fn init_global_logger(config: &AppConfig) -> Arc<Mutex<Logger>> {
|
||||
let sinks = sinks::create_sinks(config).await;
|
||||
let logger = Arc::new(Mutex::new(start_logger(config, sinks)));
|
||||
GLOBAL_LOGGER.set(logger.clone()).expect("Logger already initialized");
|
||||
logger
|
||||
}
|
||||
|
||||
/// Get the global logger instance
|
||||
///
|
||||
/// This function returns a reference to the global logger instance.
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global logger instance
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::get_global_logger;
|
||||
///
|
||||
/// let logger = get_global_logger();
|
||||
/// ```
|
||||
pub fn get_global_logger() -> &'static Arc<Mutex<Logger>> {
|
||||
GLOBAL_LOGGER.get().expect("Logger not initialized")
|
||||
}
|
||||
|
||||
/// Log information
|
||||
/// This function logs information messages.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::log_info;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_info("This is an information message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_info(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::INFO).await
|
||||
}
|
||||
|
||||
/// Log error
|
||||
/// This function logs error messages.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::log_error;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_error("This is an error message", "example").await;
|
||||
/// }
|
||||
pub async fn log_error(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::ERROR).await
|
||||
}
|
||||
|
||||
/// Log warning
|
||||
/// This function logs warning messages.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::log_warn;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_warn("This is a warning message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_warn(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::WARN).await
|
||||
}
|
||||
|
||||
/// Log debug
|
||||
/// This function logs debug messages.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::log_debug;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_debug("This is a debug message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_debug(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::DEBUG).await
|
||||
}
|
||||
|
||||
/// Log trace
|
||||
/// This function logs trace messages.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::log_trace;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_trace("This is a trace message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_trace(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::TRACE).await
|
||||
}
|
||||
|
||||
/// Log with context information
|
||||
/// This function logs messages with context information.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// - `level`: Log level
|
||||
/// - `request_id`: Request ID
|
||||
/// - `user_id`: User ID
|
||||
/// - `fields`: Additional fields
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use tracing_core::Level;
|
||||
/// use rustfs_obs::log_with_context;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_with_context("This is an information message", "example", Level::INFO, Some("req-12345".to_string()), Some("user-6789".to_string()), vec![("endpoint".to_string(), "/api/v1/data".to_string())]).await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_with_context(
|
||||
message: &str,
|
||||
source: &str,
|
||||
level: Level,
|
||||
request_id: Option<String>,
|
||||
user_id: Option<String>,
|
||||
fields: Vec<(String, String)>,
|
||||
) -> Result<(), GlobalError> {
|
||||
get_global_logger()
|
||||
.lock()
|
||||
.await
|
||||
.write_with_context(message, source, level, request_id, user_id, fields)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Log initialization status
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct InitLogStatus {
|
||||
pub timestamp: SystemTime,
|
||||
pub service_name: String,
|
||||
pub version: String,
|
||||
pub environment: String,
|
||||
}
|
||||
|
||||
impl Default for InitLogStatus {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
timestamp: SystemTime::now(),
|
||||
service_name: String::from(APP_NAME),
|
||||
version: SERVICE_VERSION.to_string(),
|
||||
environment: ENVIRONMENT.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl InitLogStatus {
|
||||
pub fn new_config(config: &OtelConfig) -> Self {
|
||||
let config = config.clone();
|
||||
let environment = config.environment.unwrap_or(ENVIRONMENT.to_string());
|
||||
let version = config.service_version.unwrap_or(SERVICE_VERSION.to_string());
|
||||
Self {
|
||||
timestamp: SystemTime::now(),
|
||||
service_name: String::from(APP_NAME),
|
||||
version,
|
||||
environment,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init_start_log(config: &OtelConfig) -> Result<(), GlobalError> {
|
||||
let status = Self::new_config(config);
|
||||
log_init_state(Some(status)).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Log initialization details during system startup
|
||||
async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), GlobalError> {
|
||||
let status = status.unwrap_or_default();
|
||||
|
||||
let base_entry = BaseLogEntry::new()
|
||||
.timestamp(chrono::DateTime::from(status.timestamp))
|
||||
.message(Some(format!(
|
||||
"Service initialization started - {} v{} in {}",
|
||||
status.service_name, status.version, status.environment
|
||||
)))
|
||||
.request_id(Some("system_init".to_string()));
|
||||
|
||||
let server_entry = ServerLogEntry::new(Level::INFO, "system_initialization".to_string())
|
||||
.with_base(base_entry)
|
||||
.user_id(Some("system".to_string()));
|
||||
|
||||
get_global_logger().lock().await.log_server_entry(server_entry).await?;
|
||||
Ok(())
|
||||
}
|
||||
@@ -1,178 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::sinks::Sink;
|
||||
use crate::{LogRecord, UnifiedLogEntry};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs::OpenOptions;
|
||||
use tokio::io;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
/// File Sink Implementation
|
||||
pub struct FileSink {
|
||||
path: String,
|
||||
buffer_size: usize,
|
||||
writer: Arc<tokio::sync::Mutex<io::BufWriter<tokio::fs::File>>>,
|
||||
entry_count: std::sync::atomic::AtomicUsize,
|
||||
last_flush: std::sync::atomic::AtomicU64,
|
||||
flush_interval_ms: u64, // Time between flushes
|
||||
flush_threshold: usize, // Number of entries before flush
|
||||
}
|
||||
|
||||
impl FileSink {
|
||||
/// Create a new FileSink instance
|
||||
pub async fn new(
|
||||
path: String,
|
||||
buffer_size: usize,
|
||||
flush_interval_ms: u64,
|
||||
flush_threshold: usize,
|
||||
) -> Result<Self, io::Error> {
|
||||
// check if the file exists
|
||||
let file_exists = tokio::fs::metadata(&path).await.is_ok();
|
||||
// if the file not exists, create it
|
||||
if !file_exists {
|
||||
tokio::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).await?;
|
||||
tracing::debug!("File does not exist, creating it. Path: {:?}", path)
|
||||
}
|
||||
let file = if file_exists {
|
||||
// If the file exists, open it in append mode
|
||||
tracing::debug!("FileSink: File exists, opening in append mode. Path: {:?}", path);
|
||||
OpenOptions::new().append(true).create(true).open(&path).await?
|
||||
} else {
|
||||
// If the file does not exist, create it
|
||||
tracing::debug!("FileSink: File does not exist, creating a new file.");
|
||||
// Create the file and write a header or initial content if needed
|
||||
OpenOptions::new().create(true).truncate(true).write(true).open(&path).await?
|
||||
};
|
||||
let writer = io::BufWriter::with_capacity(buffer_size, file);
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
Ok(FileSink {
|
||||
path,
|
||||
buffer_size,
|
||||
writer: Arc::new(tokio::sync::Mutex::new(writer)),
|
||||
entry_count: std::sync::atomic::AtomicUsize::new(0),
|
||||
last_flush: std::sync::atomic::AtomicU64::new(now),
|
||||
flush_interval_ms,
|
||||
flush_threshold,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
async fn initialize_writer(&mut self) -> io::Result<()> {
|
||||
let file = tokio::fs::File::create(&self.path).await?;
|
||||
|
||||
// Use buffer_size to create a buffer writer with a specified capacity
|
||||
let buf_writer = io::BufWriter::with_capacity(self.buffer_size, file);
|
||||
|
||||
// Replace the original writer with the new Mutex
|
||||
self.writer = Arc::new(tokio::sync::Mutex::new(buf_writer));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Get the current buffer size
|
||||
#[allow(dead_code)]
|
||||
pub fn buffer_size(&self) -> usize {
|
||||
self.buffer_size
|
||||
}
|
||||
|
||||
// How to dynamically adjust the buffer size
|
||||
#[allow(dead_code)]
|
||||
pub async fn set_buffer_size(&mut self, new_size: usize) -> io::Result<()> {
|
||||
if self.buffer_size != new_size {
|
||||
self.buffer_size = new_size;
|
||||
// Reinitialize the writer directly, without checking is_some()
|
||||
self.initialize_writer().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Check if flushing is needed based on count or time
|
||||
fn should_flush(&self) -> bool {
|
||||
// Check entry count threshold
|
||||
if self.entry_count.load(std::sync::atomic::Ordering::Relaxed) >= self.flush_threshold {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check time threshold
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
|
||||
let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
|
||||
now - last >= self.flush_interval_ms
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Sink for FileSink {
|
||||
async fn write(&self, entry: &UnifiedLogEntry) {
|
||||
let line = format!("{entry:?}\n");
|
||||
let mut writer = self.writer.lock().await;
|
||||
|
||||
if let Err(e) = writer.write_all(line.as_bytes()).await {
|
||||
eprintln!(
|
||||
"Failed to write log to file {}: {},entry timestamp:{:?}",
|
||||
self.path,
|
||||
e,
|
||||
entry.get_timestamp()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Only flush periodically to improve performance
|
||||
// Logic to determine when to flush could be added here
|
||||
// Increment the entry count
|
||||
self.entry_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
// Check if we should flush
|
||||
if self.should_flush() {
|
||||
if let Err(e) = writer.flush().await {
|
||||
eprintln!("Failed to flush log file {}: {}", self.path, e);
|
||||
return;
|
||||
}
|
||||
|
||||
// Reset counters
|
||||
self.entry_count.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
|
||||
self.last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FileSink {
|
||||
fn drop(&mut self) {
|
||||
let writer = self.writer.clone();
|
||||
let path = self.path.clone();
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
rt.block_on(async {
|
||||
let mut writer = writer.lock().await;
|
||||
if let Err(e) = writer.flush().await {
|
||||
eprintln!("Failed to flush log file {path}: {e}");
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -1,179 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::sinks::Sink;
|
||||
use crate::{LogRecord, UnifiedLogEntry};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Kafka Sink Implementation
|
||||
pub struct KafkaSink {
|
||||
producer: rdkafka::producer::FutureProducer,
|
||||
topic: String,
|
||||
batch_size: usize,
|
||||
batch_timeout_ms: u64,
|
||||
entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
|
||||
last_flush: Arc<std::sync::atomic::AtomicU64>,
|
||||
}
|
||||
|
||||
impl KafkaSink {
|
||||
/// Create a new KafkaSink instance
|
||||
pub fn new(producer: rdkafka::producer::FutureProducer, topic: String, batch_size: usize, batch_timeout_ms: u64) -> Self {
|
||||
// Create Arc-wrapped values first
|
||||
let entries = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(batch_size)));
|
||||
let last_flush = Arc::new(std::sync::atomic::AtomicU64::new(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64,
|
||||
));
|
||||
let sink = KafkaSink {
|
||||
producer: producer.clone(),
|
||||
topic: topic.clone(),
|
||||
batch_size,
|
||||
batch_timeout_ms,
|
||||
entries: entries.clone(),
|
||||
last_flush: last_flush.clone(),
|
||||
};
|
||||
|
||||
// Start background flusher
|
||||
tokio::spawn(Self::periodic_flush(producer, topic, entries, last_flush, batch_timeout_ms));
|
||||
|
||||
sink
|
||||
}
|
||||
|
||||
/// Add a getter method to read the batch_timeout_ms field
|
||||
#[allow(dead_code)]
|
||||
pub fn batch_timeout(&self) -> u64 {
|
||||
self.batch_timeout_ms
|
||||
}
|
||||
|
||||
/// Add a method to dynamically adjust the timeout if needed
|
||||
#[allow(dead_code)]
|
||||
pub fn set_batch_timeout(&mut self, new_timeout_ms: u64) {
|
||||
self.batch_timeout_ms = new_timeout_ms;
|
||||
}
|
||||
|
||||
async fn periodic_flush(
|
||||
producer: rdkafka::producer::FutureProducer,
|
||||
topic: String,
|
||||
entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
|
||||
last_flush: Arc<std::sync::atomic::AtomicU64>,
|
||||
timeout_ms: u64,
|
||||
) {
|
||||
loop {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(timeout_ms / 2)).await;
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
|
||||
let last = last_flush.load(std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
if now - last >= timeout_ms {
|
||||
let mut batch = entries.lock().await;
|
||||
if !batch.is_empty() {
|
||||
Self::send_batch(&producer, &topic, batch.drain(..).collect()).await;
|
||||
last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_batch(producer: &rdkafka::producer::FutureProducer, topic: &str, entries: Vec<UnifiedLogEntry>) {
|
||||
for entry in entries {
|
||||
let payload = match serde_json::to_string(&entry) {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
eprintln!("Failed to serialize log entry: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let span_id = entry.get_timestamp().to_rfc3339();
|
||||
|
||||
let _ = producer
|
||||
.send(
|
||||
rdkafka::producer::FutureRecord::to(topic).payload(&payload).key(&span_id),
|
||||
std::time::Duration::from_secs(5),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Sink for KafkaSink {
|
||||
async fn write(&self, entry: &UnifiedLogEntry) {
|
||||
let mut batch = self.entries.lock().await;
|
||||
batch.push(entry.clone());
|
||||
|
||||
let should_flush_by_size = batch.len() >= self.batch_size;
|
||||
let should_flush_by_time = {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
|
||||
now - last >= self.batch_timeout_ms
|
||||
};
|
||||
|
||||
if should_flush_by_size || should_flush_by_time {
|
||||
// Existing flush logic
|
||||
let entries_to_send: Vec<UnifiedLogEntry> = batch.drain(..).collect();
|
||||
let producer = self.producer.clone();
|
||||
let topic = self.topic.clone();
|
||||
|
||||
self.last_flush.store(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64,
|
||||
std::sync::atomic::Ordering::Relaxed,
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
KafkaSink::send_batch(&producer, &topic, entries_to_send).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for KafkaSink {
|
||||
fn drop(&mut self) {
|
||||
// Perform any necessary cleanup here
|
||||
// For example, you might want to flush any remaining entries
|
||||
let producer = self.producer.clone();
|
||||
let topic = self.topic.clone();
|
||||
let entries = self.entries.clone();
|
||||
let last_flush = self.last_flush.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut batch = entries.lock().await;
|
||||
if !batch.is_empty() {
|
||||
KafkaSink::send_batch(&producer, &topic, batch.drain(..).collect()).await;
|
||||
last_flush.store(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64,
|
||||
std::sync::atomic::Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
eprintln!("Dropping KafkaSink with topic: {0}", self.topic);
|
||||
}
|
||||
}
|
||||
@@ -1,123 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{AppConfig, SinkConfig, UnifiedLogEntry};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "file")]
|
||||
mod file;
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
mod kafka;
|
||||
#[cfg(feature = "webhook")]
|
||||
mod webhook;
|
||||
|
||||
/// Sink Trait definition, asynchronously write logs
|
||||
#[async_trait]
|
||||
pub trait Sink: Send + Sync {
|
||||
async fn write(&self, entry: &UnifiedLogEntry);
|
||||
}
|
||||
|
||||
/// Create a list of Sink instances
|
||||
pub async fn create_sinks(config: &AppConfig) -> Vec<Arc<dyn Sink>> {
|
||||
let mut sinks: Vec<Arc<dyn Sink>> = Vec::new();
|
||||
|
||||
for sink_config in &config.sinks {
|
||||
match sink_config {
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
SinkConfig::Kafka(kafka_config) => {
|
||||
match rdkafka::config::ClientConfig::new()
|
||||
.set("bootstrap.servers", &kafka_config.brokers)
|
||||
.set("message.timeout.ms", "5000")
|
||||
.create()
|
||||
{
|
||||
Ok(producer) => {
|
||||
sinks.push(Arc::new(kafka::KafkaSink::new(
|
||||
producer,
|
||||
kafka_config.topic.clone(),
|
||||
kafka_config
|
||||
.batch_size
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_KAFKA_BATCH_SIZE),
|
||||
kafka_config
|
||||
.batch_timeout_ms
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS),
|
||||
)));
|
||||
tracing::info!("Kafka sink created for topic: {}", kafka_config.topic);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to create Kafka producer: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "webhook")]
|
||||
SinkConfig::Webhook(webhook_config) => {
|
||||
sinks.push(Arc::new(webhook::WebhookSink::new(
|
||||
webhook_config.endpoint.clone(),
|
||||
webhook_config.auth_token.clone(),
|
||||
webhook_config
|
||||
.max_retries
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_WEBHOOK_MAX_RETRIES),
|
||||
webhook_config
|
||||
.retry_delay_ms
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS),
|
||||
)));
|
||||
tracing::info!("Webhook sink created for endpoint: {}", webhook_config.endpoint);
|
||||
}
|
||||
#[cfg(feature = "file")]
|
||||
SinkConfig::File(file_config) => {
|
||||
tracing::debug!("FileSink: Using path: {}", file_config.path);
|
||||
match file::FileSink::new(
|
||||
std::path::Path::new(&file_config.path)
|
||||
.join(rustfs_config::DEFAULT_SINK_FILE_LOG_FILE)
|
||||
.to_string_lossy()
|
||||
.to_string(),
|
||||
file_config
|
||||
.buffer_size
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_FILE_BUFFER_SIZE),
|
||||
file_config
|
||||
.flush_interval_ms
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS),
|
||||
file_config
|
||||
.flush_threshold
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_FILE_FLUSH_THRESHOLD),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(sink) => {
|
||||
sinks.push(Arc::new(sink));
|
||||
tracing::info!("File sink created for path: {}", file_config.path);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to create File sink: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(any(not(feature = "kafka"), not(target_os = "linux")))]
|
||||
SinkConfig::Kafka(_) => {
|
||||
tracing::warn!("Kafka sink is configured but the 'kafka' feature is not enabled");
|
||||
}
|
||||
#[cfg(not(feature = "webhook"))]
|
||||
SinkConfig::Webhook(_) => {
|
||||
tracing::warn!("Webhook sink is configured but the 'webhook' feature is not enabled");
|
||||
}
|
||||
#[cfg(not(feature = "file"))]
|
||||
SinkConfig::File(_) => {
|
||||
tracing::warn!("File sink is configured but the 'file' feature is not enabled");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sinks
|
||||
}
|
||||
@@ -1,84 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::UnifiedLogEntry;
|
||||
use crate::sinks::Sink;
|
||||
use async_trait::async_trait;
|
||||
|
||||
/// Webhook Sink Implementation
|
||||
pub struct WebhookSink {
|
||||
endpoint: String,
|
||||
auth_token: String,
|
||||
client: reqwest::Client,
|
||||
max_retries: usize,
|
||||
retry_delay_ms: u64,
|
||||
}
|
||||
|
||||
impl WebhookSink {
|
||||
pub fn new(endpoint: String, auth_token: String, max_retries: usize, retry_delay_ms: u64) -> Self {
|
||||
WebhookSink {
|
||||
endpoint,
|
||||
auth_token,
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap_or_else(|_| reqwest::Client::new()),
|
||||
max_retries,
|
||||
retry_delay_ms,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Sink for WebhookSink {
|
||||
async fn write(&self, entry: &UnifiedLogEntry) {
|
||||
let mut retries = 0;
|
||||
let url = self.endpoint.clone();
|
||||
let entry_clone = entry.clone();
|
||||
let auth_value = reqwest::header::HeaderValue::from_str(format!("Bearer {}", self.auth_token.clone()).as_str()).unwrap();
|
||||
while retries < self.max_retries {
|
||||
match self
|
||||
.client
|
||||
.post(&url)
|
||||
.header(reqwest::header::AUTHORIZATION, auth_value.clone())
|
||||
.json(&entry_clone)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(response) if response.status().is_success() => {
|
||||
return;
|
||||
}
|
||||
_ => {
|
||||
retries += 1;
|
||||
if retries < self.max_retries {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(
|
||||
self.retry_delay_ms * (1 << retries), // Exponential backoff
|
||||
))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!("Failed to send log to webhook after {0} retries", self.max_retries);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WebhookSink {
|
||||
fn drop(&mut self) {
|
||||
// Perform any necessary cleanup here
|
||||
// For example, you might want to log that the sink is being dropped
|
||||
eprintln!("Dropping WebhookSink with URL: {0}", self.endpoint);
|
||||
}
|
||||
}
|
||||
@@ -12,7 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::OtelConfig;
|
||||
use crate::config::OtelConfig;
|
||||
use flexi_logger::{
|
||||
Age, Cleanup, Criterion, DeferredNow, FileSpec, LogSpecification, Naming, Record, WriteMode,
|
||||
WriteMode::{AsyncWith, BufferAndFlush},
|
||||
@@ -63,7 +63,8 @@ use tracing_subscriber::{EnvFilter, Layer, layer::SubscriberExt, util::Subscribe
|
||||
/// - The tracer provider (for distributed tracing)
|
||||
/// - The meter provider (for metrics collection)
|
||||
/// - The logger provider (for structured logging)
|
||||
// Implement Debug trait correctly, rather than using derive, as some fields may not have implemented Debug
|
||||
///
|
||||
/// Implement Debug trait correctly, rather than using derive, as some fields may not have implemented Debug
|
||||
pub struct OtelGuard {
|
||||
tracer_provider: Option<SdkTracerProvider>,
|
||||
meter_provider: Option<SdkMeterProvider>,
|
||||
|
||||
@@ -29,6 +29,7 @@ documentation = "https://docs.rs/rustfs-policy/latest/rustfs_policy/"
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["constants","opa"] }
|
||||
tokio.workspace = true
|
||||
time = { workspace = true, features = ["serde-human-readable"] }
|
||||
serde = { workspace = true, features = ["derive", "rc"] }
|
||||
@@ -41,6 +42,10 @@ rand.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
jsonwebtoken = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
reqwest.workspace = true
|
||||
chrono.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
test-case.workspace = true
|
||||
temp-env = { workspace = true }
|
||||
|
||||
@@ -17,6 +17,7 @@ mod doc;
|
||||
mod effect;
|
||||
mod function;
|
||||
mod id;
|
||||
pub mod opa;
|
||||
#[allow(clippy::module_inception)]
|
||||
mod policy;
|
||||
mod principal;
|
||||
|
||||
288
crates/policy/src/policy/opa.rs
Normal file
288
crates/policy/src/policy/opa.rs
Normal file
@@ -0,0 +1,288 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::policy::Args as PArgs;
|
||||
use rustfs_config::{ENV_PREFIX, opa::*};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use std::{collections::HashMap, env, time::Duration};
|
||||
use tracing::{error, info};
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Args {
|
||||
pub url: String,
|
||||
pub auth_token: String,
|
||||
}
|
||||
impl Args {
|
||||
pub fn enable(&self) -> bool {
|
||||
!self.url.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AuthZPlugin {
|
||||
client: reqwest::Client,
|
||||
args: Args,
|
||||
}
|
||||
|
||||
fn check() -> Result<(), String> {
|
||||
let env_list = env::vars();
|
||||
let mut candidate = HashMap::new();
|
||||
let prefix = format!("{}{}", ENV_PREFIX, POLICY_PLUGIN_SUB_SYS).to_uppercase();
|
||||
for (key, value) in env_list {
|
||||
if key.starts_with(&prefix) {
|
||||
candidate.insert(key.to_string(), value);
|
||||
}
|
||||
}
|
||||
|
||||
//check required env vars
|
||||
if candidate.remove(ENV_POLICY_PLUGIN_OPA_URL).is_none() {
|
||||
return Err(format!("Missing required env var: {}", ENV_POLICY_PLUGIN_OPA_URL));
|
||||
}
|
||||
|
||||
// check optional env vars
|
||||
candidate.remove(ENV_POLICY_PLUGIN_AUTH_TOKEN);
|
||||
if !candidate.is_empty() {
|
||||
return Err(format!("Invalid env vars: {:?}", candidate));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn validate(config: &Args) -> Result<(), String> {
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
match client.post(&config.url).send().await {
|
||||
Ok(resp) => {
|
||||
match resp.status() {
|
||||
reqwest::StatusCode::OK => {
|
||||
info!("OPA is ready to accept requests.");
|
||||
}
|
||||
_ => {
|
||||
return Err(format!("OPA returned an error: {}", resp.status()));
|
||||
}
|
||||
};
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(format!("Error connecting to OPA: {}", err));
|
||||
}
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn lookup_config() -> Result<Args, String> {
|
||||
let args = Args::default();
|
||||
|
||||
let get_cfg =
|
||||
|cfg: &str| -> Result<String, String> { env::var(cfg).map_err(|e| format!("Error getting env var {}: {:?}", cfg, e)) };
|
||||
|
||||
let url = get_cfg(ENV_POLICY_PLUGIN_OPA_URL);
|
||||
if url.is_err() {
|
||||
info!("OPA is not enabled.");
|
||||
return Ok(args);
|
||||
}
|
||||
check()?;
|
||||
let args = Args {
|
||||
url: url.ok().unwrap(),
|
||||
auth_token: get_cfg(ENV_POLICY_PLUGIN_AUTH_TOKEN).unwrap_or_default(),
|
||||
};
|
||||
validate(&args).await?;
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
impl AuthZPlugin {
|
||||
pub fn new(config: Args) -> Self {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(5))
|
||||
.connect_timeout(Duration::from_secs(1))
|
||||
.pool_max_idle_per_host(10)
|
||||
.pool_idle_timeout(Some(Duration::from_secs(60)))
|
||||
.tcp_keepalive(Some(Duration::from_secs(30)))
|
||||
.tcp_nodelay(true)
|
||||
.http2_keep_alive_interval(Some(Duration::from_secs(30)))
|
||||
.http2_keep_alive_timeout(Duration::from_secs(15))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
Self { client, args: config }
|
||||
}
|
||||
|
||||
pub async fn is_allowed(&self, args: &PArgs<'_>) -> bool {
|
||||
let payload = self.build_opa_input(args);
|
||||
|
||||
let mut request = self.client.post(self.args.url.clone()).json(&payload);
|
||||
if !self.args.auth_token.is_empty() {
|
||||
request = request.header("Authorization", format!("Bearer {}", self.args.auth_token));
|
||||
}
|
||||
|
||||
match request.send().await {
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
error!("OPA returned non-success status: {}", status);
|
||||
return false;
|
||||
}
|
||||
|
||||
match resp.json::<OpaResponseEnum>().await {
|
||||
Ok(response_enum) => match response_enum {
|
||||
OpaResponseEnum::SimpleResult(result) => result.result,
|
||||
OpaResponseEnum::AllowResult(response) => response.result.allow,
|
||||
},
|
||||
Err(err) => {
|
||||
error!("Error parsing OPA response: {:?}", err);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Error sending request to OPA: {:?}", err);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_opa_input(&self, args: &PArgs<'_>) -> serde_json::Value {
|
||||
let groups = match args.groups {
|
||||
Some(g) => g.clone(),
|
||||
None => vec![],
|
||||
};
|
||||
let action_str: &str = (&args.action).into();
|
||||
json!({
|
||||
// Core authorization parameters for OPA policy evaluation
|
||||
"input":{
|
||||
"identity": {
|
||||
"account": args.account,
|
||||
"groups": groups,
|
||||
"is_owner": args.is_owner,
|
||||
"claims": args.claims
|
||||
},
|
||||
|
||||
"resource": {
|
||||
"bucket": args.bucket,
|
||||
"object": args.object,
|
||||
"arn": if args.object.is_empty() {
|
||||
format!("arn:aws:s3:::{}", args.bucket)
|
||||
} else {
|
||||
format!("arn:aws:s3:::{}/{}", args.bucket, args.object)
|
||||
}
|
||||
},
|
||||
|
||||
"action": action_str,
|
||||
|
||||
"context": {
|
||||
"conditions": args.conditions,
|
||||
"deny_only": args.deny_only,
|
||||
"timestamp": chrono::Utc::now().to_rfc3339()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct OpaResultAllow {
|
||||
allow: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct OpaResult {
|
||||
result: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct OpaResponse {
|
||||
result: OpaResultAllow,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum OpaResponseEnum {
|
||||
SimpleResult(OpaResult),
|
||||
AllowResult(OpaResponse),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use temp_env;
|
||||
|
||||
#[test]
|
||||
fn test_check_valid_config() {
|
||||
// Use temp_env to temporarily set environment variables
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", Some("http://localhost:8181/v1/data/rustfs/authz/allow")),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", Some("test-token")),
|
||||
],
|
||||
|| {
|
||||
assert!(check().is_ok());
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_missing_required_env() {
|
||||
temp_env::with_var_unset("RUSTFS_POLICY_PLUGIN_URL", || {
|
||||
temp_env::with_var("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", Some("test-token"), || {
|
||||
let result = check();
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains("Missing required env var"));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_invalid_env_vars() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", Some("http://localhost:8181/v1/data/rustfs/authz/allow")),
|
||||
("RUSTFS_POLICY_PLUGIN_INVALID", Some("invalid-value")),
|
||||
],
|
||||
|| {
|
||||
let result = check();
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains("Invalid env vars"));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_lookup_config_not_enabled() {
|
||||
temp_env::with_var_unset("RUSTFS_POLICY_PLUGIN_URL", || {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
let result = rt.block_on(async { lookup_config().await });
|
||||
|
||||
// Should return the default empty Args
|
||||
assert!(result.is_ok());
|
||||
let args = result.unwrap();
|
||||
assert!(!args.enable());
|
||||
assert_eq!(args.url, "");
|
||||
assert_eq!(args.auth_token, "");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_args_enable() {
|
||||
// Test Args enable method
|
||||
let args_enabled = Args {
|
||||
url: "http://localhost:8181".to_string(),
|
||||
auth_token: "token".to_string(),
|
||||
};
|
||||
assert!(args_enabled.enable());
|
||||
|
||||
let args_disabled = Args {
|
||||
url: "".to_string(),
|
||||
auth_token: "".to_string(),
|
||||
};
|
||||
assert!(!args_disabled.enable());
|
||||
}
|
||||
}
|
||||
@@ -17,7 +17,6 @@ use crate::error::{Error, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
/// DEFAULT_VERSION is the default version.
|
||||
/// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html
|
||||
pub const DEFAULT_VERSION: &str = "2012-10-17";
|
||||
|
||||
@@ -102,7 +102,7 @@ pub fn load_all_certs_from_directory(
|
||||
let path = entry.path();
|
||||
|
||||
if path.is_dir() {
|
||||
let domain_name = path
|
||||
let domain_name: &str = path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.ok_or_else(|| certs_error(format!("invalid domain name directory:{path:?}")))?;
|
||||
|
||||
@@ -13,18 +13,14 @@
|
||||
// limitations under the License.
|
||||
|
||||
use bytes::Bytes;
|
||||
use futures::pin_mut;
|
||||
use futures::{Stream, StreamExt};
|
||||
use std::io::Error;
|
||||
use futures::{Stream, StreamExt, pin_mut};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
fmt::Display,
|
||||
net::{IpAddr, SocketAddr, TcpListener, ToSocketAddrs},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use std::{
|
||||
net::Ipv6Addr,
|
||||
io::Error,
|
||||
net::{IpAddr, Ipv6Addr, SocketAddr, TcpListener, ToSocketAddrs},
|
||||
sync::{Arc, LazyLock, Mutex, RwLock},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{error, info};
|
||||
use transform_stream::AsyncTryStream;
|
||||
|
||||
106
docs/ansible/REAEME.md
Normal file
106
docs/ansible/REAEME.md
Normal file
@@ -0,0 +1,106 @@
|
||||
# Install rustfs with mnmd mode using ansible
|
||||
|
||||
This chapter show how to install rustfs with mnmd(multiple nodes multiple disks) using ansible playbook.Two installation method are available, namely binary and docker compose.
|
||||
|
||||
## Requirements
|
||||
|
||||
- Multiple nodes(At least 4 nodes,each has private IP and public IP)
|
||||
- Multiple disks(At least 1 disk per nodes, 4 disks is a better choice)
|
||||
- Ansible should be available
|
||||
- Docker should be available(only for docker compose installation)
|
||||
|
||||
## Binary installation and uninstallation
|
||||
|
||||
### Installation
|
||||
|
||||
For binary installation([script installation](https://rustfs.com/en/download/),you should modify the below part of the playbook,
|
||||
|
||||
```
|
||||
- name: Modify Target Server's hosts file
|
||||
blockinfile:
|
||||
path: /etc/hosts
|
||||
block: |
|
||||
172.92.20.199 rustfs-node1
|
||||
172.92.20.200 rustfs-node2
|
||||
172.92.20.201 rustfs-node3
|
||||
172.92.20.202 rustfs-node4
|
||||
```
|
||||
|
||||
Replacing the IP with your nodes' **private IP**.If you have more than 4 nodes, adding the ip in order.
|
||||
|
||||
Running the command to install rustfs
|
||||
|
||||
```
|
||||
ansible-playbook --skip-tags rustfs_uninstall binary-mnmd.yml
|
||||
```
|
||||
|
||||
After installation success, you can access the rustfs cluster via any node's public ip and 9000 port. Both default username and password are `rustfsadmin`.
|
||||
|
||||
|
||||
### Uninstallation
|
||||
|
||||
Running the command to uninstall rustfs
|
||||
|
||||
```
|
||||
ansible-playbook --tags rustfs_uninstall binary-mnmd.yml
|
||||
```
|
||||
|
||||
## Docker compose installation and uninstallation
|
||||
|
||||
**NOTE**: For docker compose installation,playbook contains docker installation task,
|
||||
|
||||
```
|
||||
tasks:
|
||||
- name: Install docker
|
||||
shell: |
|
||||
apt-get remove -y docker docker.io containerd runc || true
|
||||
apt-get update -y
|
||||
apt-get install -y ca-certificates curl gnupg lsb-release
|
||||
install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
|
||||
chmod a+r /etc/apt/keyrings/docker.gpg
|
||||
echo \
|
||||
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://mirrors.aliyun.com/docker-ce/linux/ubuntu \
|
||||
$(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
|
||||
apt-get update -y
|
||||
apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
|
||||
become: yes
|
||||
register: docker_installation_result
|
||||
changed_when: false
|
||||
|
||||
- name: Installation check
|
||||
debug:
|
||||
var: docker_installation_result.stdout
|
||||
```
|
||||
|
||||
If your node already has docker environment,you can add `tags` in the playbook and skip this task in the follow installation.By the way, the docker installation only for `Ubuntu` OS,if you have the different OS,you should modify this task as well.
|
||||
|
||||
For docker compose installation,you should also modify the below part of the playbook,
|
||||
|
||||
```
|
||||
extra_hosts:
|
||||
- "rustfs-node1:172.20.92.202"
|
||||
- "rustfs-node2:172.20.92.201"
|
||||
- "rustfs-node3:172.20.92.200"
|
||||
- "rustfs-node4:172.20.92.199"
|
||||
```
|
||||
|
||||
Replacing the IP with your nodes' **private IP**.If you have more than 4 nodes, adding the ip in order.
|
||||
|
||||
Running the command to install rustfs,
|
||||
|
||||
```
|
||||
ansible-playbook --skip-tags docker_uninstall docker-compose-mnmd.yml
|
||||
```
|
||||
|
||||
After installation success, you can access the rustfs cluster via any node's public ip and 9000 port. Both default username and password are `rustfsadmin`.
|
||||
|
||||
### Uninstallation
|
||||
|
||||
Running the command to uninstall rustfs
|
||||
|
||||
```
|
||||
ansible-playbook --tags docker_uninstall docker-compose-mnmd.yml
|
||||
```
|
||||
|
||||
|
||||
168
docs/ansible/binary-mnmd.yml
Normal file
168
docs/ansible/binary-mnmd.yml
Normal file
@@ -0,0 +1,168 @@
|
||||
---
|
||||
- name: Prepare for RustFS installation
|
||||
hosts: rustfs
|
||||
become: yes
|
||||
vars:
|
||||
ansible_python_interpreter: /usr/bin/python3
|
||||
remote_user: root
|
||||
|
||||
tasks:
|
||||
- name: Create Workspace
|
||||
file:
|
||||
path: /opt/rustfs
|
||||
state: directory
|
||||
mode: '0755'
|
||||
register: create_dir_result
|
||||
|
||||
- name: Dir Creation Result Check
|
||||
debug:
|
||||
msg: "RustFS dir created successfully"
|
||||
when: create_dir_result.changed
|
||||
|
||||
- name: Modify Target Server's hosts file
|
||||
blockinfile:
|
||||
path: /etc/hosts
|
||||
block: |
|
||||
127.0.0.1 localhost
|
||||
172.20.92.199 rustfs-node1
|
||||
172.20.92.200 rustfs-node2
|
||||
172.20.92.201 rustfs-node3
|
||||
172.20.92.202 rustfs-node4
|
||||
|
||||
- name: Create rustfs group
|
||||
group:
|
||||
name: rustfs
|
||||
system: yes
|
||||
state: present
|
||||
|
||||
- name: Create rustfs user
|
||||
user:
|
||||
name: rustfs
|
||||
shell: /bin/bash
|
||||
system: yes
|
||||
create_home: no
|
||||
group: rustfs
|
||||
state: present
|
||||
|
||||
- name: Get rustfs user id
|
||||
command: id -u rustfs
|
||||
register: rustfs_uid
|
||||
changed_when: false
|
||||
ignore_errors: yes
|
||||
|
||||
- name: Check rustfs user id
|
||||
debug:
|
||||
msg: "rustfs uid is {{ rustfs_uid.stdout }}"
|
||||
|
||||
- name: Create volume dir
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: directory
|
||||
owner: rustfs
|
||||
group: rustfs
|
||||
mode: '0755'
|
||||
loop:
|
||||
- /data/rustfs0
|
||||
- /data/rustfs1
|
||||
- /data/rustfs2
|
||||
- /data/rustfs3
|
||||
- /var/logs/rustfs
|
||||
|
||||
- name: Install RustFS
|
||||
hosts: rustfs
|
||||
become: yes
|
||||
vars:
|
||||
ansible_python_interpreter: /usr/bin/python3
|
||||
install_script_url: "https://rustfs.com/install_rustfs.sh"
|
||||
install_script_tmp: "/opt/rustfs/install_rustfs.sh"
|
||||
tags: rustfs_install
|
||||
|
||||
tasks:
|
||||
- name: Prepare configuration file
|
||||
copy:
|
||||
dest: /etc/default/rustfs
|
||||
content: |
|
||||
RUSTFS_ACCESS_KEY=rustfsadmin
|
||||
RUSTFS_SECRET_KEY=rustfsadmin
|
||||
RUSTFS_VOLUMES="http://rustfs-node{1...4}:9000/data/rustfs{0...3}"
|
||||
RUSTFS_ADDRESS=":9000"
|
||||
RUSTFS_CONSOLE_ENABLE=true
|
||||
RUST_LOG=error
|
||||
RUSTFS_OBS_LOG_DIRECTORY="/var/logs/rustfs/"
|
||||
RUSTFS_EXTERNAL_ADDRESS=0.0.0.0:9000
|
||||
owner: root
|
||||
group: root
|
||||
mode: '0644'
|
||||
|
||||
- name: Install unzip
|
||||
apt:
|
||||
name: unzip
|
||||
state: present
|
||||
update_cache: yes
|
||||
|
||||
- name: Download the rustfs install script
|
||||
get_url:
|
||||
url: "{{ install_script_url }}"
|
||||
dest: "{{ install_script_tmp }}"
|
||||
mode: '0755'
|
||||
|
||||
- name: Run rustfs installation script
|
||||
expect:
|
||||
command: bash "{{install_script_tmp}}"
|
||||
responses:
|
||||
'.*Enter your choice.*': "1\n"
|
||||
'.*Please enter RustFS service port.*': "9000\n"
|
||||
'.*Please enter RustFS console port.*': "9001\n"
|
||||
'.*Please enter data storage directory.*': "http://rustfs-node{1...4}:9000/data/rustfs{0...3}\n"
|
||||
timeout: 300
|
||||
register: rustfs_install_result
|
||||
tags:
|
||||
- rustfs_install
|
||||
|
||||
- name: Debug installation output
|
||||
debug:
|
||||
var: rustfs_install_result.stdout_lines
|
||||
|
||||
- name: Installation confirmation
|
||||
command: rustfs --version
|
||||
register: rustfs_version
|
||||
changed_when: false
|
||||
failed_when: rustfs_version.rc != 0
|
||||
|
||||
- name: Show rustfs version
|
||||
debug:
|
||||
msg: "RustFS version is {{ rustfs_version.stdout }}"
|
||||
|
||||
- name: Uninstall RustFS
|
||||
hosts: rustfs
|
||||
become: yes
|
||||
vars:
|
||||
install_script_tmp: /opt/rustfs/install_rustfs.sh
|
||||
ansible_python_interpreter: /usr/bin/python3
|
||||
tags: rustfs_uninstall
|
||||
|
||||
tasks:
|
||||
- name: Run rustfs uninstall script
|
||||
expect:
|
||||
command: bash "{{ install_script_tmp }}"
|
||||
responses:
|
||||
'Enter your choice.*': "2\n"
|
||||
'Are you sure you want to uninstall RustFS.*': "y\n"
|
||||
timeout: 300
|
||||
register: rustfs_uninstall_result
|
||||
tags: rustfs_uninstall
|
||||
|
||||
- name: Debug uninstall output
|
||||
debug:
|
||||
var: rustfs_uninstall_result.stdout_lines
|
||||
|
||||
- name: Delete data dir
|
||||
file:
|
||||
path: "{{ item }}"
|
||||
state: absent
|
||||
loop:
|
||||
- /data/rustfs0
|
||||
- /data/rustfs1
|
||||
- /data/rustfs2
|
||||
- /data/rustfs3
|
||||
- /var/logs/rustfs
|
||||
115
docs/ansible/docker-compose-mnmd.yml
Normal file
115
docs/ansible/docker-compose-mnmd.yml
Normal file
@@ -0,0 +1,115 @@
|
||||
---
|
||||
- name: Prepare for RustFS installation
|
||||
hosts: rustfs
|
||||
become: yes
|
||||
vars:
|
||||
ansible_python_interpreter: /usr/bin/python3
|
||||
install_script_url: "https://rustfs.com/install_rustfs.sh"
|
||||
docker_compose: "/opt/rustfs/docker-compose"
|
||||
remote_user: root
|
||||
|
||||
tasks:
|
||||
- name: Install docker
|
||||
tags: docker_install
|
||||
shell: |
|
||||
apt-get remove -y docker docker.io containerd runc || true
|
||||
apt-get update -y
|
||||
apt-get install -y ca-certificates curl gnupg lsb-release
|
||||
install -m 0755 -d /etc/apt/keyrings
|
||||
curl -fsSL https://mirrors.aliyun.com/docker-ce/linux/ubuntu/gpg | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
|
||||
chmod a+r /etc/apt/keyrings/docker.gpg
|
||||
echo \
|
||||
"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://mirrors.aliyun.com/docker-ce/linux/ubuntu \
|
||||
$(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null
|
||||
apt-get update -y
|
||||
apt-get install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin
|
||||
become: yes
|
||||
register: docker_installation_result
|
||||
changed_when: false
|
||||
when: ansible_facts['distribution'] == "Ubuntu"
|
||||
|
||||
- name: Installation check
|
||||
debug:
|
||||
var: docker_installation_result.stdout
|
||||
when: ansible_facts['distribution'] == "Ubuntu"
|
||||
|
||||
- name: Create docker compose dir
|
||||
file:
|
||||
path: "{{ docker_compose }}"
|
||||
state: directory
|
||||
mode: '0755'
|
||||
|
||||
- name: Prepare docker compose file
|
||||
copy:
|
||||
content: |
|
||||
services:
|
||||
rustfs:
|
||||
image: rustfs/rustfs:latest
|
||||
container_name: rustfs
|
||||
hostname: rustfs
|
||||
network_mode: host
|
||||
environment:
|
||||
# Use service names and correct disk indexing (1..4 to match mounted paths)
|
||||
- RUSTFS_VOLUMES=http://rustfs-node{1...4}:9000/data/rustfs{1...4}
|
||||
- RUSTFS_ADDRESS=0.0.0.0:9000
|
||||
- RUSTFS_CONSOLE_ENABLE=true
|
||||
- RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
|
||||
- RUSTFS_EXTERNAL_ADDRESS=0.0.0.0:9000 # Same as internal since no port mapping
|
||||
- RUSTFS_ACCESS_KEY=rustfsadmin
|
||||
- RUSTFS_SECRET_KEY=rustfsadmin
|
||||
- RUSTFS_CMD=rustfs
|
||||
command: ["sh", "-c", "sleep 3 && rustfs"]
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"curl -f http://localhost:9000/health && curl -f http://localhost:9001/health || exit 1"
|
||||
]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 3
|
||||
start_period: 30s
|
||||
ports:
|
||||
- "9000:9000" # API endpoint
|
||||
- "9001:9001" # Console
|
||||
volumes:
|
||||
- rustfs-data1:/data/rustfs1
|
||||
- rustfs-data2:/data/rustfs2
|
||||
- rustfs-data3:/data/rustfs3
|
||||
- rustfs-data4:/data/rustfs4
|
||||
extra_hosts:
|
||||
- "rustfs-node1:172.20.92.202"
|
||||
- "rustfs-node2:172.20.92.201"
|
||||
- "rustfs-node3:172.20.92.200"
|
||||
- "rustfs-node4:172.20.92.199"
|
||||
|
||||
volumes:
|
||||
rustfs-data1:
|
||||
rustfs-data2:
|
||||
rustfs-data3:
|
||||
rustfs-data4:
|
||||
|
||||
dest: "{{ docker_compose }}/docker-compose.yml"
|
||||
mode: '0644'
|
||||
|
||||
- name: Install rustfs using docker compose
|
||||
tags: rustfs_install
|
||||
command: docker compose -f "{{ docker_compose}}/docker-compose.yml" up -d
|
||||
args:
|
||||
chdir: "{{ docker_compose }}"
|
||||
|
||||
- name: Get docker compose output
|
||||
command: docker compose ps
|
||||
args:
|
||||
chdir: "{{ docker_compose }}"
|
||||
register: docker_compose_output
|
||||
|
||||
- name: Check the docker compose installation output
|
||||
debug:
|
||||
msg: "{{ docker_compose_output.stdout }}"
|
||||
|
||||
- name: Uninstall rustfs using docker compose
|
||||
tags: rustfs_uninstall
|
||||
command: docker compose -f "{{ docker_compose}}/docker-compose.yml" down
|
||||
args:
|
||||
chdir: "{{ docker_compose }}"
|
||||
@@ -9,6 +9,7 @@ This directory contains practical deployment examples and configurations for Rus
|
||||
Complete Docker Compose example for deploying RustFS in a 4-node, 4-drive-per-node configuration.
|
||||
|
||||
**Features:**
|
||||
|
||||
- Proper disk indexing (1..4) to avoid VolumeNotFound errors
|
||||
- Startup coordination via `wait-and-start.sh` script
|
||||
- Service discovery using Docker service names
|
||||
@@ -18,25 +19,47 @@ Complete Docker Compose example for deploying RustFS in a 4-node, 4-drive-per-no
|
||||
**Use Case:** Production-ready multi-node deployment for high availability and performance.
|
||||
|
||||
**Quick Start:**
|
||||
|
||||
```bash
|
||||
cd docs/examples/mnmd
|
||||
docker-compose up -d
|
||||
```
|
||||
|
||||
**See also:**
|
||||
|
||||
- [MNMD README](./mnmd/README.md) - Detailed usage guide
|
||||
- [MNMD CHECKLIST](./mnmd/CHECKLIST.md) - Step-by-step verification
|
||||
|
||||
## Other Deployment Examples
|
||||
|
||||
For additional deployment examples, see:
|
||||
- [`examples/`](/examples/) - Root-level examples directory with:
|
||||
- `docker-quickstart.sh` - Quick start script for basic deployments
|
||||
- `enhanced-docker-deployment.sh` - Advanced deployment scenarios
|
||||
- `docker-comprehensive.yml` - Docker Compose with multiple profiles
|
||||
|
||||
- [`docker/`](./docker/) - Root-level examples directory with:
|
||||
- `docker-quickstart.sh` - Quick start script for basic deployments, Quickstart script (basic
|
||||
/dev/prod/status/test/cleanup)
|
||||
- `enhanced-docker-deployment.sh` - Advanced deployment scenarios, Advanced deployment script with multiple
|
||||
scenarios and detailed logs (basic /dev/prod/all/status/test/logs/cleanup)
|
||||
- `enhanced-security-deployment.sh` - Production-ready scripts with TLS, throttling, and secure credential
|
||||
generation
|
||||
- `docker-comprehensive.yml` - Docker Compose with multiple profiles, Docker Compose files containing multiple
|
||||
profiles (basic / dev / production / enterprise / api-only / nginx, etc.)
|
||||
- Usage example:
|
||||
```bash
|
||||
# Rapid development environment
|
||||
./docs/examples/docker/docker-quickstart.sh dev
|
||||
|
||||
# Start dev profile using Docker Compose
|
||||
docker-compose -f docs/examples/docker/docker-comprehensive.yml --profile dev up -d
|
||||
|
||||
# Secure deployment
|
||||
./docs/examples/docker/enhanced-security-deployment.sh
|
||||
```
|
||||
- Note: If the original CI or other documents refer to the old path `examples/`, please update it to
|
||||
`docs/examples/docker/`. Relative links within the document are already in this README.
|
||||
|
||||
- [`.docker/compose/`](/.docker/compose/) - Docker Compose configurations:
|
||||
- `docker-compose.cluster.yaml` - Basic cluster setup
|
||||
- `docker-compose.observability.yaml` - Observability stack integration
|
||||
- `docker-compose.cluster.yaml` - Basic cluster setup
|
||||
- `docker-compose.observability.yaml` - Observability stack integration
|
||||
|
||||
## Related Documentation
|
||||
|
||||
@@ -47,6 +70,7 @@ For additional deployment examples, see:
|
||||
## Contributing
|
||||
|
||||
When adding new examples:
|
||||
|
||||
1. Create a dedicated subdirectory under `docs/examples/`
|
||||
2. Include a comprehensive README.md
|
||||
3. Provide working configuration files
|
||||
@@ -56,5 +80,6 @@ When adding new examples:
|
||||
## Support
|
||||
|
||||
For issues or questions:
|
||||
|
||||
- GitHub Issues: https://github.com/rustfs/rustfs/issues
|
||||
- Documentation: https://rustfs.io
|
||||
- Documentation: https://rustfs.com/docs
|
||||
|
||||
@@ -1,10 +1,12 @@
|
||||
# RustFS Docker Deployment Examples
|
||||
|
||||
This directory contains various deployment scripts and configuration files for RustFS with console and endpoint service separation.
|
||||
This directory contains various deployment scripts and configuration files for RustFS with console and endpoint service
|
||||
separation.
|
||||
|
||||
## Quick Start Scripts
|
||||
|
||||
### `docker-quickstart.sh`
|
||||
|
||||
The fastest way to get RustFS running with different configurations.
|
||||
|
||||
```bash
|
||||
@@ -28,6 +30,7 @@ The fastest way to get RustFS running with different configurations.
|
||||
```
|
||||
|
||||
### `enhanced-docker-deployment.sh`
|
||||
|
||||
Comprehensive deployment script with multiple scenarios and detailed logging.
|
||||
|
||||
```bash
|
||||
@@ -51,7 +54,9 @@ Comprehensive deployment script with multiple scenarios and detailed logging.
|
||||
```
|
||||
|
||||
### `enhanced-security-deployment.sh`
|
||||
Production-ready deployment with enhanced security features including TLS, rate limiting, and secure credential generation.
|
||||
|
||||
Production-ready deployment with enhanced security features including TLS, rate limiting, and secure credential
|
||||
generation.
|
||||
|
||||
```bash
|
||||
# Deploy with security hardening
|
||||
@@ -68,6 +73,7 @@ Production-ready deployment with enhanced security features including TLS, rate
|
||||
## Docker Compose Examples
|
||||
|
||||
### `docker-comprehensive.yml`
|
||||
|
||||
Complete Docker Compose configuration with multiple deployment profiles.
|
||||
|
||||
```bash
|
||||
@@ -106,6 +112,7 @@ docker-compose -f docker-comprehensive.yml --profile dev up -d
|
||||
```
|
||||
|
||||
**Access Points:**
|
||||
|
||||
- API: http://localhost:9010 (or 9030 for enhanced)
|
||||
- Console: http://localhost:9011/rustfs/console/ (or 9031 for enhanced)
|
||||
- Credentials: dev-admin / dev-secret
|
||||
@@ -121,6 +128,7 @@ docker-compose -f docker-comprehensive.yml --profile dev up -d
|
||||
```
|
||||
|
||||
**Features:**
|
||||
|
||||
- TLS encryption for console
|
||||
- Rate limiting enabled
|
||||
- Restricted CORS policies
|
||||
@@ -194,13 +202,13 @@ curl http://localhost:9001/health
|
||||
|
||||
### Port Mappings
|
||||
|
||||
| Deployment | API Port | Console Port | Description |
|
||||
|-----------|----------|--------------|-------------|
|
||||
| Basic | 9000 | 9001 | Simple deployment |
|
||||
| Dev | 9010 | 9011 | Development environment |
|
||||
| Prod | 9020 | 9021 | Production-like setup |
|
||||
| Enterprise | 9030 | 9443 | Enterprise with TLS |
|
||||
| API-Only | 9040 | - | API endpoint only |
|
||||
| Deployment | API Port | Console Port | Description |
|
||||
|------------|----------|--------------|-------------------------|
|
||||
| Basic | 9000 | 9001 | Simple deployment |
|
||||
| Dev | 9010 | 9011 | Development environment |
|
||||
| Prod | 9020 | 9021 | Production-like setup |
|
||||
| Enterprise | 9030 | 9443 | Enterprise with TLS |
|
||||
| API-Only | 9040 | - | API endpoint only |
|
||||
|
||||
### Network Isolation
|
||||
|
||||
@@ -213,11 +221,13 @@ Production deployments use network isolation:
|
||||
## Security Considerations
|
||||
|
||||
### Development
|
||||
|
||||
- Permissive CORS policies for easy testing
|
||||
- Debug logging enabled
|
||||
- Default credentials for simplicity
|
||||
|
||||
### Production
|
||||
### Production
|
||||
|
||||
- Restrictive CORS policies
|
||||
- TLS encryption for console
|
||||
- Rate limiting enabled
|
||||
@@ -226,6 +236,7 @@ Production deployments use network isolation:
|
||||
- Network isolation
|
||||
|
||||
### Enterprise
|
||||
|
||||
- Complete TLS encryption
|
||||
- Advanced rate limiting
|
||||
- Authentication timeouts
|
||||
@@ -260,11 +271,12 @@ docker exec rustfs-container netstat -tulpn | grep -E ':(9000|9001)'
|
||||
|
||||
## Migration from Previous Versions
|
||||
|
||||
See [docs/console-separation.md](../docs/console-separation.md) for detailed migration instructions from single-port deployments to the separated architecture.
|
||||
See [docs/console-separation.md](../../console-separation.md) for detailed migration instructions from single-port
|
||||
deployments to the separated architecture.
|
||||
|
||||
## Additional Resources
|
||||
|
||||
- [Console Separation Documentation](../docs/console-separation.md)
|
||||
- [Docker Compose Configuration](../docker-compose.yml)
|
||||
- [Main Dockerfile](../Dockerfile)
|
||||
- [Security Best Practices](../docs/console-separation.md#security-hardening)
|
||||
- [Console Separation Documentation](../../console-separation.md)
|
||||
- [Docker Compose Configuration](../../../docker-compose.yml)
|
||||
- [Main Dockerfile](../../../Dockerfile)
|
||||
- [Security Best Practices](../../console-separation.md#security-hardening)
|
||||
@@ -57,6 +57,7 @@ rustfs-protos = { workspace = true }
|
||||
rustfs-s3select-query = { workspace = true }
|
||||
rustfs-targets = { workspace = true }
|
||||
rustfs-kms = { workspace = true }
|
||||
rustfs-lock.workspace = true
|
||||
atoi = { workspace = true }
|
||||
atomic_enum = { workspace = true }
|
||||
axum.workspace = true
|
||||
@@ -69,7 +70,9 @@ chrono = { workspace = true }
|
||||
clap = { workspace = true }
|
||||
datafusion = { workspace = true }
|
||||
const-str = { workspace = true }
|
||||
flatbuffers.workspace = true
|
||||
futures.workspace = true
|
||||
futures-util.workspace = true
|
||||
hyper.workspace = true
|
||||
hyper-util.workspace = true
|
||||
http.workspace = true
|
||||
@@ -80,6 +83,7 @@ mime_guess = { workspace = true }
|
||||
opentelemetry = { workspace = true }
|
||||
pin-project-lite.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
rmp-serde.workspace = true
|
||||
rustls = { workspace = true }
|
||||
rust-embed = { workspace = true, features = ["interpolate-folder-path"] }
|
||||
s3s.workspace = true
|
||||
@@ -116,10 +120,7 @@ url = { workspace = true }
|
||||
urlencoding = { workspace = true }
|
||||
uuid = { workspace = true }
|
||||
zip = { workspace = true }
|
||||
futures-util.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
rustfs-lock.workspace = true
|
||||
|
||||
|
||||
[target.'cfg(any(target_os = "macos", target_os = "freebsd", target_os = "netbsd", target_os = "openbsd"))'.dependencies]
|
||||
sysctl = { workspace = true }
|
||||
@@ -129,7 +130,7 @@ sysctl = { workspace = true }
|
||||
libsystemd.workspace = true
|
||||
|
||||
[target.'cfg(all(target_os = "linux", target_env = "gnu"))'.dependencies]
|
||||
tikv-jemallocator = "0.6"
|
||||
tikv-jemallocator = "0.6.1"
|
||||
|
||||
[target.'cfg(all(target_os = "linux", target_env = "musl"))'.dependencies]
|
||||
mimalloc = "0.1"
|
||||
|
||||
@@ -127,7 +127,7 @@ async fn async_main() -> Result<()> {
|
||||
init_license(opt.license.clone());
|
||||
|
||||
// Initialize Observability
|
||||
let (_logger, guard) = init_obs(Some(opt.clone().obs_endpoint)).await;
|
||||
let guard = init_obs(Some(opt.clone().obs_endpoint)).await;
|
||||
|
||||
// print startup logo
|
||||
info!("{}", LOGO);
|
||||
|
||||
@@ -49,7 +49,7 @@ use rustfs_ecstore::{
|
||||
versioning::VersioningApi,
|
||||
versioning_sys::BucketVersioningSys,
|
||||
},
|
||||
client::object_api_utils::format_etag,
|
||||
client::object_api_utils::to_s3s_etag,
|
||||
compress::{MIN_COMPRESSIBLE_SIZE, is_compressible},
|
||||
disk::{error::DiskError, error_reduce::is_all_buckets_not_found},
|
||||
error::{StorageError, is_err_bucket_not_found, is_err_object_not_found, is_err_version_not_found},
|
||||
@@ -117,7 +117,7 @@ use tokio::{io::AsyncRead, sync::mpsc};
|
||||
use tokio_stream::wrappers::ReceiverStream;
|
||||
use tokio_tar::Archive;
|
||||
use tokio_util::io::{ReaderStream, StreamReader};
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing::{debug, error, info, instrument, warn};
|
||||
use uuid::Uuid;
|
||||
|
||||
macro_rules! try_ {
|
||||
@@ -437,7 +437,7 @@ impl FS {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let e_tag = _obj_info.etag.clone().map(|etag| format_etag(&etag));
|
||||
let e_tag = _obj_info.etag.clone().map(|etag| to_s3s_etag(&etag));
|
||||
|
||||
// // store.put_object(bucket, object, data, opts);
|
||||
|
||||
@@ -505,7 +505,7 @@ async fn get_validated_store(bucket: &str) -> S3Result<Arc<rustfs_ecstore::store
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl S3 for FS {
|
||||
#[tracing::instrument(
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
skip(self, req),
|
||||
fields(start_time=?time::OffsetDateTime::now_utc())
|
||||
@@ -538,7 +538,7 @@ impl S3 for FS {
|
||||
let event_args = rustfs_notify::event::EventArgs {
|
||||
event_name: EventName::BucketCreated,
|
||||
bucket_name: bucket.clone(),
|
||||
object: rustfs_ecstore::store_api::ObjectInfo { ..Default::default() },
|
||||
object: ObjectInfo { ..Default::default() },
|
||||
req_params: rustfs_utils::extract_req_params_header(&req.headers),
|
||||
resp_elements: rustfs_utils::extract_resp_elements(&S3Response::new(output.clone())),
|
||||
version_id: String::new(),
|
||||
@@ -555,7 +555,7 @@ impl S3 for FS {
|
||||
}
|
||||
|
||||
/// Copy an object from one location to another
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn copy_object(&self, req: S3Request<CopyObjectInput>) -> S3Result<S3Response<CopyObjectOutput>> {
|
||||
let CopyObjectInput {
|
||||
copy_source,
|
||||
@@ -709,7 +709,7 @@ impl S3 for FS {
|
||||
src_info.put_object_reader = Some(PutObjReader::new(reader));
|
||||
|
||||
// check quota
|
||||
// TODO: src metadada
|
||||
// TODO: src metadata
|
||||
|
||||
for (k, v) in compress_metadata {
|
||||
src_info.user_defined.insert(k, v);
|
||||
@@ -725,7 +725,7 @@ impl S3 for FS {
|
||||
// warn!("copy_object oi {:?}", &oi);
|
||||
let object_info = oi.clone();
|
||||
let copy_object_result = CopyObjectResult {
|
||||
e_tag: oi.etag.map(|etag| format_etag(&etag)),
|
||||
e_tag: oi.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
last_modified: oi.mod_time.map(Timestamp::from),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -923,7 +923,7 @@ impl S3 for FS {
|
||||
}
|
||||
|
||||
/// Delete a bucket
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn delete_bucket(&self, req: S3Request<DeleteBucketInput>) -> S3Result<S3Response<DeleteBucketOutput>> {
|
||||
let input = req.input;
|
||||
// TODO: DeleteBucketInput doesn't have force parameter?
|
||||
@@ -945,7 +945,7 @@ impl S3 for FS {
|
||||
let event_args = rustfs_notify::event::EventArgs {
|
||||
event_name: EventName::BucketRemoved,
|
||||
bucket_name: input.bucket,
|
||||
object: rustfs_ecstore::store_api::ObjectInfo { ..Default::default() },
|
||||
object: ObjectInfo { ..Default::default() },
|
||||
req_params: rustfs_utils::extract_req_params_header(&req.headers),
|
||||
resp_elements: rustfs_utils::extract_resp_elements(&S3Response::new(DeleteBucketOutput {})),
|
||||
version_id: String::new(),
|
||||
@@ -962,7 +962,7 @@ impl S3 for FS {
|
||||
}
|
||||
|
||||
/// Delete an object
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn delete_object(&self, mut req: S3Request<DeleteObjectInput>) -> S3Result<S3Response<DeleteObjectOutput>> {
|
||||
let DeleteObjectInput {
|
||||
bucket, key, version_id, ..
|
||||
@@ -1069,7 +1069,7 @@ impl S3 for FS {
|
||||
let event_args = rustfs_notify::event::EventArgs {
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
object: rustfs_ecstore::store_api::ObjectInfo {
|
||||
object: ObjectInfo {
|
||||
name: key.clone(),
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
@@ -1090,7 +1090,7 @@ impl S3 for FS {
|
||||
}
|
||||
|
||||
/// Delete multiple objects
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn delete_objects(&self, req: S3Request<DeleteObjectsInput>) -> S3Result<S3Response<DeleteObjectsOutput>> {
|
||||
let DeleteObjectsInput { bucket, delete, .. } = req.input;
|
||||
|
||||
@@ -1329,7 +1329,7 @@ impl S3 for FS {
|
||||
let event_args = rustfs_notify::event::EventArgs {
|
||||
event_name,
|
||||
bucket_name: bucket.clone(),
|
||||
object: rustfs_ecstore::store_api::ObjectInfo {
|
||||
object: ObjectInfo {
|
||||
name: dobj.object_name,
|
||||
bucket: bucket.clone(),
|
||||
..Default::default()
|
||||
@@ -1350,7 +1350,7 @@ impl S3 for FS {
|
||||
}
|
||||
|
||||
/// Get bucket location
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn get_bucket_location(&self, req: S3Request<GetBucketLocationInput>) -> S3Result<S3Response<GetBucketLocationOutput>> {
|
||||
// mc get 1
|
||||
let input = req.input;
|
||||
@@ -1375,7 +1375,7 @@ impl S3 for FS {
|
||||
}
|
||||
|
||||
/// Get bucket notification
|
||||
#[tracing::instrument(
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
skip(self, req),
|
||||
fields(start_time=?time::OffsetDateTime::now_utc())
|
||||
@@ -1435,9 +1435,9 @@ impl S3 for FS {
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let info = reader.object_info;
|
||||
tracing::debug!(object_size = info.size, part_count = info.parts.len(), "GET object metadata snapshot");
|
||||
debug!(object_size = info.size, part_count = info.parts.len(), "GET object metadata snapshot");
|
||||
for part in &info.parts {
|
||||
tracing::debug!(
|
||||
debug!(
|
||||
part_number = part.number,
|
||||
part_size = part.size,
|
||||
part_actual_size = part.actual_size,
|
||||
@@ -1487,7 +1487,7 @@ impl S3 for FS {
|
||||
let mut managed_encryption_applied = false;
|
||||
let mut managed_original_size: Option<i64> = None;
|
||||
|
||||
tracing::debug!(
|
||||
debug!(
|
||||
"GET object metadata check: stored_sse_algorithm={:?}, stored_sse_key_md5={:?}, provided_sse_key={:?}",
|
||||
stored_sse_algorithm,
|
||||
stored_sse_key_md5,
|
||||
@@ -1501,20 +1501,16 @@ impl S3 for FS {
|
||||
// Each part needs to be decrypted individually, which requires storage layer changes
|
||||
// Note: Single part objects also have info.parts.len() == 1, but they are not true multipart uploads
|
||||
if info.parts.len() > 1 {
|
||||
tracing::warn!(
|
||||
warn!(
|
||||
"SSE-C multipart object detected with {} parts. Currently, multipart SSE-C upload parts are not encrypted during upload_part, so no decryption is needed during GET.",
|
||||
info.parts.len()
|
||||
);
|
||||
|
||||
// Verify that the provided key MD5 matches the stored MD5 for security
|
||||
if let Some(stored_md5) = stored_sse_key_md5 {
|
||||
tracing::debug!("SSE-C MD5 comparison: provided='{}', stored='{}'", sse_key_md5_provided, stored_md5);
|
||||
debug!("SSE-C MD5 comparison: provided='{}', stored='{}'", sse_key_md5_provided, stored_md5);
|
||||
if sse_key_md5_provided != stored_md5 {
|
||||
tracing::error!(
|
||||
"SSE-C key MD5 mismatch: provided='{}', stored='{}'",
|
||||
sse_key_md5_provided,
|
||||
stored_md5
|
||||
);
|
||||
error!("SSE-C key MD5 mismatch: provided='{}', stored='{}'", sse_key_md5_provided, stored_md5);
|
||||
return Err(
|
||||
ApiError::from(StorageError::other("SSE-C key does not match object encryption key")).into()
|
||||
);
|
||||
@@ -1532,13 +1528,9 @@ impl S3 for FS {
|
||||
} else {
|
||||
// Verify that the provided key MD5 matches the stored MD5
|
||||
if let Some(stored_md5) = stored_sse_key_md5 {
|
||||
tracing::debug!("SSE-C MD5 comparison: provided='{}', stored='{}'", sse_key_md5_provided, stored_md5);
|
||||
debug!("SSE-C MD5 comparison: provided='{}', stored='{}'", sse_key_md5_provided, stored_md5);
|
||||
if sse_key_md5_provided != stored_md5 {
|
||||
tracing::error!(
|
||||
"SSE-C key MD5 mismatch: provided='{}', stored='{}'",
|
||||
sse_key_md5_provided,
|
||||
stored_md5
|
||||
);
|
||||
error!("SSE-C key MD5 mismatch: provided='{}', stored='{}'", sse_key_md5_provided, stored_md5);
|
||||
return Err(
|
||||
ApiError::from(StorageError::other("SSE-C key does not match object encryption key")).into()
|
||||
);
|
||||
@@ -1613,14 +1605,13 @@ impl S3 for FS {
|
||||
let response_content_length = if stored_sse_algorithm.is_some() {
|
||||
if let Some(original_size_str) = info.user_defined.get("x-amz-server-side-encryption-customer-original-size") {
|
||||
let original_size = original_size_str.parse::<i64>().unwrap_or(content_length);
|
||||
tracing::info!(
|
||||
info!(
|
||||
"SSE-C decryption: using original size {} instead of encrypted size {}",
|
||||
original_size,
|
||||
content_length
|
||||
original_size, content_length
|
||||
);
|
||||
original_size
|
||||
} else {
|
||||
tracing::debug!("SSE-C decryption: no original size found, using content_length {}", content_length);
|
||||
debug!("SSE-C decryption: no original size found, using content_length {}", content_length);
|
||||
content_length
|
||||
}
|
||||
} else if managed_encryption_applied {
|
||||
@@ -1629,7 +1620,7 @@ impl S3 for FS {
|
||||
content_length
|
||||
};
|
||||
|
||||
tracing::info!("Final response_content_length: {}", response_content_length);
|
||||
info!("Final response_content_length: {}", response_content_length);
|
||||
|
||||
if stored_sse_algorithm.is_some() || managed_encryption_applied {
|
||||
let limit_reader = HardLimitReader::new(Box::new(WarpReader::new(final_stream)), response_content_length);
|
||||
@@ -1639,7 +1630,7 @@ impl S3 for FS {
|
||||
// For SSE-C encrypted objects, don't use bytes_stream to limit the stream
|
||||
// because DecryptReader needs to read all encrypted data to produce decrypted output
|
||||
let body = if stored_sse_algorithm.is_some() || managed_encryption_applied {
|
||||
tracing::info!("Managed SSE: Using unlimited stream for decryption");
|
||||
info!("Managed SSE: Using unlimited stream for decryption");
|
||||
Some(StreamingBlob::wrap(ReaderStream::with_capacity(final_stream, DEFAULT_READ_BUFFER_SIZE)))
|
||||
} else {
|
||||
Some(StreamingBlob::wrap(bytes_stream(
|
||||
@@ -1670,7 +1661,7 @@ impl S3 for FS {
|
||||
content_type,
|
||||
accept_ranges: Some("bytes".to_string()),
|
||||
content_range,
|
||||
e_tag: info.etag.map(|etag| format_etag(&etag)),
|
||||
e_tag: info.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
metadata: Some(info.user_defined),
|
||||
server_side_encryption,
|
||||
sse_customer_algorithm,
|
||||
@@ -1702,7 +1693,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn head_bucket(&self, req: S3Request<HeadBucketInput>) -> S3Result<S3Response<HeadBucketOutput>> {
|
||||
let input = req.input;
|
||||
|
||||
@@ -1719,7 +1710,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(HeadBucketOutput::default()))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn head_object(&self, req: S3Request<HeadObjectInput>) -> S3Result<S3Response<HeadObjectOutput>> {
|
||||
// mc get 2
|
||||
let HeadObjectInput {
|
||||
@@ -1804,7 +1795,7 @@ impl S3 for FS {
|
||||
content_length: Some(content_length),
|
||||
content_type,
|
||||
last_modified,
|
||||
e_tag: info.etag.map(|etag| format_etag(&etag)),
|
||||
e_tag: info.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
metadata: Some(metadata),
|
||||
version_id: info.version_id.map(|v| v.to_string()),
|
||||
server_side_encryption,
|
||||
@@ -1838,7 +1829,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn list_buckets(&self, req: S3Request<ListBucketsInput>) -> S3Result<S3Response<ListBucketsOutput>> {
|
||||
// mc ls
|
||||
|
||||
@@ -1854,20 +1845,26 @@ impl S3 for FS {
|
||||
.await
|
||||
.is_err()
|
||||
{
|
||||
bucket_infos.retain(|info| {
|
||||
let req_info = req.extensions.get_mut::<ReqInfo>().expect("ReqInfo not found");
|
||||
bucket_infos = futures::stream::iter(bucket_infos)
|
||||
.filter_map(|info| async {
|
||||
let mut req_clone = req.clone();
|
||||
let req_info = req_clone.extensions.get_mut::<ReqInfo>().expect("ReqInfo not found");
|
||||
req_info.bucket = Some(info.name.clone());
|
||||
|
||||
req_info.bucket = Some(info.name.clone());
|
||||
|
||||
futures::executor::block_on(async {
|
||||
authorize_request(&mut req, Action::S3Action(S3Action::ListBucketAction))
|
||||
if authorize_request(&mut req_clone, Action::S3Action(S3Action::ListBucketAction))
|
||||
.await
|
||||
.is_ok()
|
||||
|| authorize_request(&mut req, Action::S3Action(S3Action::GetBucketLocationAction))
|
||||
|| authorize_request(&mut req_clone, Action::S3Action(S3Action::GetBucketLocationAction))
|
||||
.await
|
||||
.is_ok()
|
||||
{
|
||||
Some(info)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
});
|
||||
.collect()
|
||||
.await;
|
||||
}
|
||||
|
||||
let buckets: Vec<Bucket> = bucket_infos
|
||||
@@ -1887,7 +1884,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn list_objects(&self, req: S3Request<ListObjectsInput>) -> S3Result<S3Response<ListObjectsOutput>> {
|
||||
let v2_resp = self.list_objects_v2(req.map_input(Into::into)).await?;
|
||||
|
||||
@@ -1903,7 +1900,7 @@ impl S3 for FS {
|
||||
}))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn list_objects_v2(&self, req: S3Request<ListObjectsV2Input>) -> S3Result<S3Response<ListObjectsV2Output>> {
|
||||
// warn!("list_objects_v2 req {:?}", &req.input);
|
||||
let ListObjectsV2Input {
|
||||
@@ -1955,7 +1952,7 @@ impl S3 for FS {
|
||||
key: Some(v.name.to_owned()),
|
||||
last_modified: v.mod_time.map(Timestamp::from),
|
||||
size: Some(v.get_actual_size().unwrap_or_default()),
|
||||
e_tag: v.etag.clone().map(|etag| format_etag(&etag)),
|
||||
e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -2034,7 +2031,7 @@ impl S3 for FS {
|
||||
size: Some(v.size),
|
||||
version_id: v.version_id.map(|v| v.to_string()),
|
||||
is_latest: Some(v.is_latest),
|
||||
e_tag: v.etag.clone().map(|etag| format_etag(&etag)),
|
||||
e_tag: v.etag.clone().map(|etag| to_s3s_etag(&etag)),
|
||||
..Default::default() // TODO: another fields
|
||||
}
|
||||
})
|
||||
@@ -2076,7 +2073,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
// #[tracing::instrument(level = "debug", skip(self, req))]
|
||||
// #[instrument(level = "debug", skip(self, req))]
|
||||
async fn put_object(&self, req: S3Request<PutObjectInput>) -> S3Result<S3Response<PutObjectOutput>> {
|
||||
if req
|
||||
.headers
|
||||
@@ -2137,17 +2134,17 @@ impl S3 for FS {
|
||||
|
||||
// TDD: Get bucket default encryption configuration
|
||||
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
|
||||
tracing::debug!("TDD: bucket_sse_config={:?}", bucket_sse_config);
|
||||
debug!("TDD: bucket_sse_config={:?}", bucket_sse_config);
|
||||
|
||||
// TDD: Determine effective encryption configuration (request overrides bucket default)
|
||||
let original_sse = server_side_encryption.clone();
|
||||
let effective_sse = server_side_encryption.or_else(|| {
|
||||
bucket_sse_config.as_ref().and_then(|(config, _timestamp)| {
|
||||
tracing::debug!("TDD: Processing bucket SSE config: {:?}", config);
|
||||
debug!("TDD: Processing bucket SSE config: {:?}", config);
|
||||
config.rules.first().and_then(|rule| {
|
||||
tracing::debug!("TDD: Processing SSE rule: {:?}", rule);
|
||||
debug!("TDD: Processing SSE rule: {:?}", rule);
|
||||
rule.apply_server_side_encryption_by_default.as_ref().map(|sse| {
|
||||
tracing::debug!("TDD: Found SSE default: {:?}", sse);
|
||||
debug!("TDD: Found SSE default: {:?}", sse);
|
||||
match sse.sse_algorithm.as_str() {
|
||||
"AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256),
|
||||
"aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
|
||||
@@ -2157,7 +2154,7 @@ impl S3 for FS {
|
||||
})
|
||||
})
|
||||
});
|
||||
tracing::debug!("TDD: effective_sse={:?} (original={:?})", effective_sse, original_sse);
|
||||
debug!("TDD: effective_sse={:?} (original={:?})", effective_sse, original_sse);
|
||||
|
||||
let mut effective_kms_key_id = ssekms_key_id.or_else(|| {
|
||||
bucket_sse_config.as_ref().and_then(|(config, _timestamp)| {
|
||||
@@ -2311,7 +2308,7 @@ impl S3 for FS {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
let event_info = obj_info.clone();
|
||||
let e_tag = obj_info.etag.clone().map(|etag| format_etag(&etag));
|
||||
let e_tag = obj_info.etag.clone().map(|etag| to_s3s_etag(&etag));
|
||||
|
||||
let repoptions =
|
||||
get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts);
|
||||
@@ -2350,7 +2347,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn create_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<CreateMultipartUploadInput>,
|
||||
@@ -2391,17 +2388,17 @@ impl S3 for FS {
|
||||
|
||||
// TDD: Get bucket SSE configuration for multipart upload
|
||||
let bucket_sse_config = metadata_sys::get_sse_config(&bucket).await.ok();
|
||||
tracing::debug!("TDD: Got bucket SSE config for multipart: {:?}", bucket_sse_config);
|
||||
debug!("TDD: Got bucket SSE config for multipart: {:?}", bucket_sse_config);
|
||||
|
||||
// TDD: Determine effective encryption (request parameters override bucket defaults)
|
||||
let original_sse = server_side_encryption.clone();
|
||||
let effective_sse = server_side_encryption.or_else(|| {
|
||||
bucket_sse_config.as_ref().and_then(|(config, _timestamp)| {
|
||||
tracing::debug!("TDD: Processing bucket SSE config for multipart: {:?}", config);
|
||||
debug!("TDD: Processing bucket SSE config for multipart: {:?}", config);
|
||||
config.rules.first().and_then(|rule| {
|
||||
tracing::debug!("TDD: Processing SSE rule for multipart: {:?}", rule);
|
||||
debug!("TDD: Processing SSE rule for multipart: {:?}", rule);
|
||||
rule.apply_server_side_encryption_by_default.as_ref().map(|sse| {
|
||||
tracing::debug!("TDD: Found SSE default for multipart: {:?}", sse);
|
||||
debug!("TDD: Found SSE default for multipart: {:?}", sse);
|
||||
match sse.sse_algorithm.as_str() {
|
||||
"AES256" => ServerSideEncryption::from_static(ServerSideEncryption::AES256),
|
||||
"aws:kms" => ServerSideEncryption::from_static(ServerSideEncryption::AWS_KMS),
|
||||
@@ -2411,7 +2408,7 @@ impl S3 for FS {
|
||||
})
|
||||
})
|
||||
});
|
||||
tracing::debug!("TDD: effective_sse for multipart={:?} (original={:?})", effective_sse, original_sse);
|
||||
debug!("TDD: effective_sse for multipart={:?} (original={:?})", effective_sse, original_sse);
|
||||
|
||||
let _original_kms_key_id = ssekms_key_id.clone();
|
||||
let mut effective_kms_key_id = ssekms_key_id.or_else(|| {
|
||||
@@ -2490,7 +2487,7 @@ impl S3 for FS {
|
||||
let event_args = rustfs_notify::event::EventArgs {
|
||||
event_name: EventName::ObjectCreatedCompleteMultipartUpload,
|
||||
bucket_name: bucket_name.clone(),
|
||||
object: rustfs_ecstore::store_api::ObjectInfo {
|
||||
object: ObjectInfo {
|
||||
name: object_name,
|
||||
bucket: bucket_name,
|
||||
..Default::default()
|
||||
@@ -2510,7 +2507,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn upload_part(&self, req: S3Request<UploadPartInput>) -> S3Result<S3Response<UploadPartOutput>> {
|
||||
let UploadPartInput {
|
||||
body,
|
||||
@@ -2576,7 +2573,7 @@ impl S3 for FS {
|
||||
.await?
|
||||
.is_some();
|
||||
|
||||
// If managed encryption will be applied and we have Content-Length, buffer the entire body
|
||||
// If managed encryption will be applied, and we have Content-Length, buffer the entire body
|
||||
// This is necessary because encryption changes the data size, which causes Content-Length mismatches
|
||||
if will_apply_managed_encryption && size.is_some() {
|
||||
let mut total = 0i64;
|
||||
@@ -2674,14 +2671,14 @@ impl S3 for FS {
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let output = UploadPartOutput {
|
||||
e_tag: info.etag.map(|etag| format_etag(&etag)),
|
||||
e_tag: info.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn upload_part_copy(&self, req: S3Request<UploadPartCopyInput>) -> S3Result<S3Response<UploadPartCopyOutput>> {
|
||||
let UploadPartCopyInput {
|
||||
bucket,
|
||||
@@ -2860,7 +2857,7 @@ impl S3 for FS {
|
||||
|
||||
// Create response
|
||||
let copy_part_result = CopyPartResult {
|
||||
e_tag: part_info.etag.map(|etag| format_etag(&etag)),
|
||||
e_tag: part_info.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
last_modified: part_info.last_mod.map(Timestamp::from),
|
||||
..Default::default()
|
||||
};
|
||||
@@ -2874,7 +2871,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn list_parts(&self, req: S3Request<ListPartsInput>) -> S3Result<S3Response<ListPartsOutput>> {
|
||||
let ListPartsInput {
|
||||
bucket,
|
||||
@@ -2913,7 +2910,7 @@ impl S3 for FS {
|
||||
res.parts
|
||||
.into_iter()
|
||||
.map(|p| Part {
|
||||
e_tag: p.etag.map(|etag| format_etag(&etag)),
|
||||
e_tag: p.etag.map(|etag| to_s3s_etag(&etag)),
|
||||
last_modified: p.last_mod.map(Timestamp::from),
|
||||
part_number: Some(p.part_num as i32),
|
||||
size: Some(p.size as i64),
|
||||
@@ -3007,7 +3004,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn complete_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<CompleteMultipartUploadInput>,
|
||||
@@ -3038,26 +3035,24 @@ impl S3 for FS {
|
||||
};
|
||||
|
||||
// TDD: Get multipart info to extract encryption configuration before completing
|
||||
tracing::info!(
|
||||
info!(
|
||||
"TDD: Attempting to get multipart info for bucket={}, key={}, upload_id={}",
|
||||
bucket,
|
||||
key,
|
||||
upload_id
|
||||
bucket, key, upload_id
|
||||
);
|
||||
let multipart_info = store
|
||||
.get_multipart_info(&bucket, &key, &upload_id, &ObjectOptions::default())
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
tracing::info!("TDD: Got multipart info successfully");
|
||||
tracing::info!("TDD: Multipart info metadata: {:?}", multipart_info.user_defined);
|
||||
info!("TDD: Got multipart info successfully");
|
||||
info!("TDD: Multipart info metadata: {:?}", multipart_info.user_defined);
|
||||
|
||||
// TDD: Extract encryption information from multipart upload metadata
|
||||
let server_side_encryption = multipart_info
|
||||
.user_defined
|
||||
.get("x-amz-server-side-encryption")
|
||||
.map(|s| ServerSideEncryption::from(s.clone()));
|
||||
tracing::info!(
|
||||
info!(
|
||||
"TDD: Raw encryption from metadata: {:?} -> parsed: {:?}",
|
||||
multipart_info.user_defined.get("x-amz-server-side-encryption"),
|
||||
server_side_encryption
|
||||
@@ -3068,10 +3063,9 @@ impl S3 for FS {
|
||||
.get("x-amz-server-side-encryption-aws-kms-key-id")
|
||||
.cloned();
|
||||
|
||||
tracing::info!(
|
||||
info!(
|
||||
"TDD: Extracted encryption info - SSE: {:?}, KMS Key: {:?}",
|
||||
server_side_encryption,
|
||||
ssekms_key_id
|
||||
server_side_encryption, ssekms_key_id
|
||||
);
|
||||
|
||||
let obj_info = store
|
||||
@@ -3080,24 +3074,22 @@ impl S3 for FS {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
tracing::info!(
|
||||
info!(
|
||||
"TDD: Creating output with SSE: {:?}, KMS Key: {:?}",
|
||||
server_side_encryption,
|
||||
ssekms_key_id
|
||||
server_side_encryption, ssekms_key_id
|
||||
);
|
||||
let output = CompleteMultipartUploadOutput {
|
||||
bucket: Some(bucket.clone()),
|
||||
key: Some(key.clone()),
|
||||
e_tag: obj_info.etag.clone().map(|etag| format_etag(&etag)),
|
||||
e_tag: obj_info.etag.clone().map(|etag| to_s3s_etag(&etag)),
|
||||
location: Some("us-east-1".to_string()),
|
||||
server_side_encryption, // TDD: Return encryption info
|
||||
ssekms_key_id, // TDD: Return KMS key ID if present
|
||||
..Default::default()
|
||||
};
|
||||
tracing::info!(
|
||||
info!(
|
||||
"TDD: Created output: SSE={:?}, KMS={:?}",
|
||||
output.server_side_encryption,
|
||||
output.ssekms_key_id
|
||||
output.server_side_encryption, output.ssekms_key_id
|
||||
);
|
||||
|
||||
let mt2 = HashMap::new();
|
||||
@@ -3110,15 +3102,14 @@ impl S3 for FS {
|
||||
warn!("need multipart replication");
|
||||
schedule_replication(obj_info, store, dsc, ReplicationType::Object).await;
|
||||
}
|
||||
tracing::info!(
|
||||
info!(
|
||||
"TDD: About to return S3Response with output: SSE={:?}, KMS={:?}",
|
||||
output.server_side_encryption,
|
||||
output.ssekms_key_id
|
||||
output.server_side_encryption, output.ssekms_key_id
|
||||
);
|
||||
Ok(S3Response::new(output))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn abort_multipart_upload(
|
||||
&self,
|
||||
req: S3Request<AbortMultipartUploadInput>,
|
||||
@@ -3140,7 +3131,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(AbortMultipartUploadOutput { ..Default::default() }))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_bucket_tagging(&self, req: S3Request<GetBucketTaggingInput>) -> S3Result<S3Response<GetBucketTaggingOutput>> {
|
||||
let bucket = req.input.bucket.clone();
|
||||
// check bucket exists.
|
||||
@@ -3163,7 +3154,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(GetBucketTaggingOutput { tag_set }))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn put_bucket_tagging(&self, req: S3Request<PutBucketTaggingInput>) -> S3Result<S3Response<PutBucketTaggingOutput>> {
|
||||
let PutBucketTaggingInput { bucket, tagging, .. } = req.input;
|
||||
|
||||
@@ -3185,7 +3176,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(Default::default()))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn delete_bucket_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketTaggingInput>,
|
||||
@@ -3199,7 +3190,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(DeleteBucketTaggingOutput {}))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self, req))]
|
||||
#[instrument(level = "debug", skip(self, req))]
|
||||
async fn put_object_tagging(&self, req: S3Request<PutObjectTaggingInput>) -> S3Result<S3Response<PutObjectTaggingOutput>> {
|
||||
let PutObjectTaggingInput {
|
||||
bucket,
|
||||
@@ -3264,7 +3255,7 @@ impl S3 for FS {
|
||||
let event_args = rustfs_notify::event::EventArgs {
|
||||
event_name: EventName::ObjectCreatedPutTagging,
|
||||
bucket_name: bucket.clone(),
|
||||
object: rustfs_ecstore::store_api::ObjectInfo {
|
||||
object: ObjectInfo {
|
||||
name: object.clone(),
|
||||
bucket,
|
||||
..Default::default()
|
||||
@@ -3284,7 +3275,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(PutObjectTaggingOutput { version_id: None }))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_object_tagging(&self, req: S3Request<GetObjectTaggingInput>) -> S3Result<S3Response<GetObjectTaggingOutput>> {
|
||||
let GetObjectTaggingInput { bucket, key: object, .. } = req.input;
|
||||
|
||||
@@ -3306,7 +3297,7 @@ impl S3 for FS {
|
||||
}))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn delete_object_tagging(
|
||||
&self,
|
||||
req: S3Request<DeleteObjectTaggingInput>,
|
||||
@@ -3331,7 +3322,7 @@ impl S3 for FS {
|
||||
let event_args = rustfs_notify::event::EventArgs {
|
||||
event_name: EventName::ObjectCreatedDeleteTagging,
|
||||
bucket_name: bucket.clone(),
|
||||
object: rustfs_ecstore::store_api::ObjectInfo {
|
||||
object: ObjectInfo {
|
||||
name: object.clone(),
|
||||
bucket,
|
||||
..Default::default()
|
||||
@@ -3351,7 +3342,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(DeleteObjectTaggingOutput { version_id: None }))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_bucket_versioning(
|
||||
&self,
|
||||
req: S3Request<GetBucketVersioningInput>,
|
||||
@@ -3374,7 +3365,7 @@ impl S3 for FS {
|
||||
}))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn put_bucket_versioning(
|
||||
&self,
|
||||
req: S3Request<PutBucketVersioningInput>,
|
||||
@@ -3531,7 +3522,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(DeleteBucketPolicyOutput {}))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_bucket_lifecycle_configuration(
|
||||
&self,
|
||||
req: S3Request<GetBucketLifecycleConfigurationInput>,
|
||||
@@ -3564,7 +3555,7 @@ impl S3 for FS {
|
||||
}))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn put_bucket_lifecycle_configuration(
|
||||
&self,
|
||||
req: S3Request<PutBucketLifecycleConfigurationInput>,
|
||||
@@ -3598,7 +3589,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(PutBucketLifecycleConfigurationOutput::default()))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn delete_bucket_lifecycle(
|
||||
&self,
|
||||
req: S3Request<DeleteBucketLifecycleInput>,
|
||||
@@ -3703,7 +3694,7 @@ impl S3 for FS {
|
||||
Ok(S3Response::new(DeleteBucketEncryptionOutput::default()))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn get_object_lock_configuration(
|
||||
&self,
|
||||
req: S3Request<GetObjectLockConfigurationInput>,
|
||||
@@ -3725,7 +3716,7 @@ impl S3 for FS {
|
||||
}))
|
||||
}
|
||||
|
||||
#[tracing::instrument(level = "debug", skip(self))]
|
||||
#[instrument(level = "debug", skip(self))]
|
||||
async fn put_object_lock_configuration(
|
||||
&self,
|
||||
req: S3Request<PutObjectLockConfigurationInput>,
|
||||
@@ -3771,7 +3762,7 @@ impl S3 for FS {
|
||||
.await
|
||||
.map_err(ApiError::from)?;
|
||||
|
||||
let rcfg = match metadata_sys::get_replication_config(&bucket).await {
|
||||
let rcfg = match get_replication_config(&bucket).await {
|
||||
Ok((cfg, _created)) => Some(cfg),
|
||||
Err(err) => {
|
||||
error!("get_replication_config err {:?}", err);
|
||||
@@ -4101,7 +4092,7 @@ impl S3 for FS {
|
||||
let event_args = rustfs_notify::event::EventArgs {
|
||||
event_name: EventName::ObjectAccessedAttributes,
|
||||
bucket_name: bucket.clone(),
|
||||
object: rustfs_ecstore::store_api::ObjectInfo {
|
||||
object: ObjectInfo {
|
||||
name: key.clone(),
|
||||
bucket,
|
||||
..Default::default()
|
||||
|
||||
@@ -68,11 +68,6 @@ export RUSTFS_OBS_LOG_POOL_CAPA=10240
|
||||
export RUSTFS_OBS_LOG_MESSAGE_CAPA=32768
|
||||
export RUSTFS_OBS_LOG_FLUSH_MS=300
|
||||
|
||||
export RUSTFS_SINKS_FILE_PATH="$current_dir/deploy/logs"
|
||||
export RUSTFS_SINKS_FILE_BUFFER_SIZE=12
|
||||
export RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS=1000
|
||||
export RUSTFS_SINKS_FILE_FLUSH_THRESHOLD=100
|
||||
|
||||
#tokio runtime
|
||||
export RUSTFS_RUNTIME_WORKER_THREADS=16
|
||||
export RUSTFS_RUNTIME_MAX_BLOCKING_THREADS=1024
|
||||
@@ -82,21 +77,6 @@ export RUSTFS_RUNTIME_THREAD_STACK_SIZE=1024*1024
|
||||
export RUSTFS_RUNTIME_THREAD_KEEP_ALIVE=60
|
||||
export RUSTFS_RUNTIME_GLOBAL_QUEUE_INTERVAL=31
|
||||
|
||||
#
|
||||
# Kafka sink 配置
|
||||
#export RUSTFS_SINKS_KAFKA_BROKERS=localhost:9092
|
||||
#export RUSTFS_SINKS_KAFKA_TOPIC=logs
|
||||
#export RUSTFS_SINKS_KAFKA_BATCH_SIZE=100
|
||||
#export RUSTFS_SINKS_KAFKA_BATCH_TIMEOUT_MS=1000
|
||||
#
|
||||
# Webhook sink 配置
|
||||
#export RUSTFS_SINKS_WEBHOOK_ENDPOINT=http://localhost:8080/webhook
|
||||
#export RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN=you-auth-token
|
||||
#export RUSTFS_SINKS_WEBHOOK_BATCH_SIZE=100
|
||||
#export RUSTFS_SINKS_WEBHOOK_BATCH_TIMEOUT_MS=1000
|
||||
#
|
||||
#export RUSTFS_LOGGER_QUEUE_CAPACITY=10
|
||||
|
||||
export OTEL_INSTRUMENTATION_NAME="rustfs"
|
||||
export OTEL_INSTRUMENTATION_VERSION="0.1.1"
|
||||
export OTEL_INSTRUMENTATION_SCHEMA_URL="https://opentelemetry.io/schemas/1.31.0"
|
||||
@@ -115,6 +95,9 @@ export RUSTFS_NOTIFY_WEBHOOK_ENABLE_MASTER="on" # Whether to enable webhook noti
|
||||
export RUSTFS_NOTIFY_WEBHOOK_ENDPOINT_MASTER="http://[::]:3020/webhook" # Webhook notification address
|
||||
export RUSTFS_NOTIFY_WEBHOOK_QUEUE_DIR_MASTER="$current_dir/deploy/logs/notify"
|
||||
|
||||
# export RUSTFS_POLICY_PLUGIN_URL="http://localhost:8181/v1/data/rustfs/authz/allow" # The URL of the OPA system
|
||||
# export RUSTFS_POLICY_PLUGIN_AUTH_TOKEN="your-opa-token" # The authentication token for the OPA system is optional
|
||||
|
||||
|
||||
export RUSTFS_NS_SCANNER_INTERVAL=60 # Object scanning interval in seconds
|
||||
# exportRUSTFS_SKIP_BACKGROUND_TASK=true
|
||||
|
||||
Reference in New Issue
Block a user