mirror of
https://github.com/rustfs/rustfs.git
synced 2026-01-17 01:30:33 +00:00
Compare commits
2 Commits
1.0.0-alph
...
feat/net-m
| Author | SHA1 | Date | |
|---|---|---|---|
|
7df97fb266 | ||
|
|
e99da872ac |
7
.vscode/launch.json
vendored
7
.vscode/launch.json
vendored
@@ -20,10 +20,7 @@
|
||||
}
|
||||
},
|
||||
"env": {
|
||||
"RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=debug",
|
||||
"RUSTFS_SKIP_BACKGROUND_TASK": "on",
|
||||
// "RUSTFS_POLICY_PLUGIN_URL":"http://localhost:8181/v1/data/rustfs/authz/allow",
|
||||
// "RUSTFS_POLICY_PLUGIN_AUTH_TOKEN":"your-opa-token"
|
||||
"RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug,iam=info"
|
||||
},
|
||||
"args": [
|
||||
"--access-key",
|
||||
@@ -32,8 +29,6 @@
|
||||
"rustfsadmin",
|
||||
"--address",
|
||||
"0.0.0.0:9010",
|
||||
"--server-domains",
|
||||
"127.0.0.1:9010",
|
||||
"./target/volume/test{1...4}"
|
||||
],
|
||||
"cwd": "${workspaceFolder}"
|
||||
|
||||
441
Cargo.lock
generated
441
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
30
Cargo.toml
30
Cargo.toml
@@ -28,7 +28,6 @@ members = [
|
||||
"crates/madmin", # Management dashboard and admin API interface
|
||||
"crates/notify", # Notification system for events
|
||||
"crates/obs", # Observability utilities
|
||||
"crates/policy", # Policy management
|
||||
"crates/protos", # Protocol buffer definitions
|
||||
"crates/rio", # Rust I/O utilities and abstractions
|
||||
"crates/targets", # Target-specific configurations and utilities
|
||||
@@ -113,19 +112,17 @@ brotli = "8.0.2"
|
||||
bytes = { version = "1.10.1", features = ["serde"] }
|
||||
bytesize = "2.1.0"
|
||||
byteorder = "1.5.0"
|
||||
cfg-if = "1.0.4"
|
||||
cfg-if = "1.0.3"
|
||||
convert_case = "0.8.0"
|
||||
crc-fast = "1.3.0"
|
||||
chacha20poly1305 = { version = "0.10.1" }
|
||||
chrono = { version = "0.4.42", features = ["serde"] }
|
||||
clap = { version = "4.5.49", features = ["derive", "env"] }
|
||||
clap = { version = "4.5.48", features = ["derive", "env"] }
|
||||
const-str = { version = "0.7.0", features = ["std", "proc"] }
|
||||
crc32fast = "1.5.0"
|
||||
crc32c = "0.6.8"
|
||||
crc64fast-nvme = "1.2.0"
|
||||
criterion = { version = "0.7", features = ["html_reports"] }
|
||||
crossbeam-queue = "0.3.12"
|
||||
datafusion = "50.2.0"
|
||||
datafusion = "50.1.0"
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.10"
|
||||
flatbuffers = "25.9.23"
|
||||
@@ -195,9 +192,10 @@ pretty_assertions = "1.4.1"
|
||||
quick-xml = "0.38.3"
|
||||
rand = "0.9.2"
|
||||
rayon = "1.11.0"
|
||||
reed-solomon-simd = { version = "3.1.0" }
|
||||
regex = { version = "1.12.2" }
|
||||
reqwest = { version = "0.12.24", default-features = false, features = [
|
||||
rdkafka = { version = "0.38.0", features = ["tokio"] }
|
||||
reed-solomon-simd = { version = "3.0.1" }
|
||||
regex = { version = "1.12.1" }
|
||||
reqwest = { version = "0.12.23", default-features = false, features = [
|
||||
"rustls-tls-webpki-roots",
|
||||
"charset",
|
||||
"http2",
|
||||
@@ -207,16 +205,16 @@ reqwest = { version = "0.12.24", default-features = false, features = [
|
||||
"blocking",
|
||||
] }
|
||||
rmcp = { version = "0.8.1" }
|
||||
rmp = { version = "0.8.14" }
|
||||
rmp-serde = { version = "1.3.0" }
|
||||
rsa = { version = "0.9.8" }
|
||||
rmp = "0.8.14"
|
||||
rmp-serde = "1.3.0"
|
||||
rsa = "0.9.8"
|
||||
rumqttc = { version = "0.25.0" }
|
||||
rust-embed = { version = "8.7.2" }
|
||||
rustc-hash = { version = "2.1.1" }
|
||||
rustls = { version = "0.23.32", features = ["ring", "logging", "std", "tls12"], default-features = false }
|
||||
rustls-pki-types = "1.12.0"
|
||||
rustls-pemfile = "2.2.0"
|
||||
s3s = { version = "0.12.0-rc.3", features = ["minio"] }
|
||||
s3s = { version = "0.12.0-rc.2", features = ["minio"] }
|
||||
schemars = "1.0.4"
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
serde_json = { version = "1.0.145", features = ["raw_value"] }
|
||||
@@ -230,7 +228,7 @@ smallvec = { version = "1.15.1", features = ["serde"] }
|
||||
smartstring = "1.0.1"
|
||||
snafu = "0.8.9"
|
||||
snap = "1.1.1"
|
||||
socket2 = "0.6.1"
|
||||
socket2 = "0.6.0"
|
||||
starshard = { version = "0.5.0", features = ["rayon", "async", "serde"] }
|
||||
strum = { version = "0.27.2", features = ["derive"] }
|
||||
sysinfo = "0.37.1"
|
||||
@@ -246,7 +244,7 @@ time = { version = "0.3.44", features = [
|
||||
"macros",
|
||||
"serde",
|
||||
] }
|
||||
tokio = { version = "1.48.0", features = ["fs", "rt-multi-thread"] }
|
||||
tokio = { version = "1.47.1", features = ["fs", "rt-multi-thread"] }
|
||||
tokio-rustls = { version = "0.26.4", default-features = false, features = ["logging", "tls12", "ring"] }
|
||||
tokio-stream = { version = "0.1.17" }
|
||||
tokio-tar = "0.3.1"
|
||||
@@ -257,7 +255,7 @@ tonic-prost = { version = "0.14.2" }
|
||||
tonic-prost-build = { version = "0.14.2" }
|
||||
tower = { version = "0.5.2", features = ["timeout"] }
|
||||
tower-http = { version = "0.6.6", features = ["cors"] }
|
||||
tracing = { version = "0.1.41" }
|
||||
tracing = "0.1.41"
|
||||
tracing-core = "0.1.34"
|
||||
tracing-error = "0.2.1"
|
||||
tracing-opentelemetry = "0.32.0"
|
||||
|
||||
@@ -58,7 +58,7 @@ LABEL name="RustFS" \
|
||||
url="https://rustfs.com" \
|
||||
license="Apache-2.0"
|
||||
|
||||
RUN apk add --no-cache ca-certificates coreutils curl
|
||||
RUN apk add --no-cache ca-certificates coreutils
|
||||
|
||||
COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
|
||||
COPY --from=build /build/rustfs /usr/bin/rustfs
|
||||
|
||||
85
README.md
85
README.md
@@ -29,11 +29,7 @@ English | <a href="https://github.com/rustfs/rustfs/blob/main/README_ZH.md">简
|
||||
<a href="https://readme-i18n.com/rustfs/rustfs?lang=ru">Русский</a>
|
||||
</p>
|
||||
|
||||
RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages
|
||||
worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature,
|
||||
support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in
|
||||
comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation,
|
||||
RustFS provides faster speed and safer distributed features for high-performance object storage.
|
||||
RustFS is a high-performance distributed object storage software built using Rust, one of the most popular languages worldwide. Along with MinIO, it shares a range of advantages such as simplicity, S3 compatibility, open-source nature, support for data lakes, AI, and big data. Furthermore, it has a better and more user-friendly open-source license in comparison to other storage systems, being constructed under the Apache license. As Rust serves as its foundation, RustFS provides faster speed and safer distributed features for high-performance object storage.
|
||||
|
||||
> ⚠️ **RustFS is under rapid development. Do NOT use in production environments!**
|
||||
|
||||
@@ -50,27 +46,27 @@ RustFS provides faster speed and safer distributed features for high-performance
|
||||
|
||||
Stress test server parameters
|
||||
|
||||
| Type | parameter | Remark |
|
||||
|---------|-----------|----------------------------------------------------------|
|
||||
| CPU | 2 Core | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz | |
|
||||
| Memory | 4GB | |
|
||||
| Network | 15Gbp | |
|
||||
| Driver | 40GB x 4 | IOPS 3800 / Driver |
|
||||
| Type | parameter | Remark |
|
||||
| - | - | - |
|
||||
|CPU | 2 Core | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz| |
|
||||
|Memory| 4GB | |
|
||||
|Network | 15Gbp | |
|
||||
|Driver | 40GB x 4 | IOPS 3800 / Driver |
|
||||
|
||||
<https://github.com/user-attachments/assets/2e4979b5-260c-4f2c-ac12-c87fd558072a>
|
||||
|
||||
### RustFS vs Other object storage
|
||||
|
||||
| RustFS | Other object storage |
|
||||
|---------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------|
|
||||
| Powerful Console | Simple and useless Console |
|
||||
| Developed based on Rust language, memory is safer | Developed in Go or C, with potential issues like memory GC/leaks |
|
||||
| Does not report logs to third-party countries | Reporting logs to other third countries may violate national security laws |
|
||||
| Licensed under Apache, more business-friendly | AGPL V3 License and other License, polluted open source and License traps, infringement of intellectual property rights |
|
||||
| Comprehensive S3 support, works with domestic and international cloud providers | Full support for S3, but no local cloud vendor support |
|
||||
| Rust-based development, strong support for secure and innovative devices | Poor support for edge gateways and secure innovative devices |
|
||||
| Stable commercial prices, free community support | High pricing, with costs up to $250,000 for 1PiB |
|
||||
| No risk | Intellectual property risks and risks of prohibited uses |
|
||||
| RustFS | Other object storage|
|
||||
| - | - |
|
||||
| Powerful Console | Simple and useless Console |
|
||||
| Developed based on Rust language, memory is safer | Developed in Go or C, with potential issues like memory GC/leaks |
|
||||
| Does not report logs to third-party countries | Reporting logs to other third countries may violate national security laws |
|
||||
| Licensed under Apache, more business-friendly | AGPL V3 License and other License, polluted open source and License traps, infringement of intellectual property rights |
|
||||
| Comprehensive S3 support, works with domestic and international cloud providers | Full support for S3, but no local cloud vendor support |
|
||||
| Rust-based development, strong support for secure and innovative devices | Poor support for edge gateways and secure innovative devices|
|
||||
| Stable commercial prices, free community support | High pricing, with costs up to $250,000 for 1PiB |
|
||||
| No risk | Intellectual property risks and risks of prohibited uses |
|
||||
|
||||
## Quickstart
|
||||
|
||||
@@ -95,16 +91,13 @@ To get started with RustFS, follow these steps:
|
||||
docker run -d -p 9000:9000 -v $(pwd)/data:/data -v $(pwd)/logs:/logs rustfs/rustfs:1.0.0.alpha.45
|
||||
```
|
||||
|
||||
For docker installation, you can also run the container with docker compose. With the `docker-compose.yml` file under
|
||||
root directory, running the command:
|
||||
For docker installation, you can also run the container with docker compose. With the `docker-compose.yml` file under root directory, running the command:
|
||||
|
||||
```
|
||||
docker compose --profile observability up -d
|
||||
```
|
||||
|
||||
**NOTE**: You should be better to have a look for `docker-compose.yaml` file. Because, several services contains in the
|
||||
file. Grafan,prometheus,jaeger containers will be launched using docker compose file, which is helpful for rustfs
|
||||
observability. If you want to start redis as well as nginx container, you can specify the corresponding profiles.
|
||||
|
||||
**NOTE**: You should be better to have a look for `docker-compose.yaml` file. Because, several services contains in the file. Grafan,prometheus,jaeger containers will be launched using docker compose file, which is helpful for rustfs observability. If you want to start redis as well as nginx container, you can specify the corresponding profiles.
|
||||
|
||||
3. **Build from Source (Option 3) - Advanced Users**
|
||||
|
||||
@@ -125,10 +118,10 @@ observability. If you want to start redis as well as nginx container, you can sp
|
||||
```
|
||||
|
||||
The `docker-buildx.sh` script supports:
|
||||
- **Multi-architecture builds**: `linux/amd64`, `linux/arm64`
|
||||
- **Automatic version detection**: Uses git tags or commit hashes
|
||||
- **Registry flexibility**: Supports Docker Hub, GitHub Container Registry, etc.
|
||||
- **Build optimization**: Includes caching and parallel builds
|
||||
- **Multi-architecture builds**: `linux/amd64`, `linux/arm64`
|
||||
- **Automatic version detection**: Uses git tags or commit hashes
|
||||
- **Registry flexibility**: Supports Docker Hub, GitHub Container Registry, etc.
|
||||
- **Build optimization**: Includes caching and parallel builds
|
||||
|
||||
You can also use Make targets for convenience:
|
||||
|
||||
@@ -139,29 +132,23 @@ observability. If you want to start redis as well as nginx container, you can sp
|
||||
make help-docker # Show all Docker-related commands
|
||||
```
|
||||
|
||||
4. **Access the Console**: Open your web browser and navigate to `http://localhost:9000` to access the RustFS console,
|
||||
default username and password is `rustfsadmin` .
|
||||
4. **Access the Console**: Open your web browser and navigate to `http://localhost:9000` to access the RustFS console, default username and password is `rustfsadmin` .
|
||||
5. **Create a Bucket**: Use the console to create a new bucket for your objects.
|
||||
6. **Upload Objects**: You can upload files directly through the console or use S3-compatible APIs to interact with your
|
||||
RustFS instance.
|
||||
6. **Upload Objects**: You can upload files directly through the console or use S3-compatible APIs to interact with your RustFS instance.
|
||||
|
||||
**NOTE**: If you want to access RustFS instance with `https`, you can refer
|
||||
to [TLS configuration docs](https://docs.rustfs.com/integration/tls-configured.html).
|
||||
**NOTE**: If you want to access RustFS instance with `https`, you can refer to [TLS configuration docs](https://docs.rustfs.com/integration/tls-configured.html).
|
||||
|
||||
## Documentation
|
||||
|
||||
For detailed documentation, including configuration options, API references, and advanced usage, please visit
|
||||
our [Documentation](https://docs.rustfs.com).
|
||||
For detailed documentation, including configuration options, API references, and advanced usage, please visit our [Documentation](https://docs.rustfs.com).
|
||||
|
||||
## Getting Help
|
||||
|
||||
If you have any questions or need assistance, you can:
|
||||
|
||||
- Check the [FAQ](https://github.com/rustfs/rustfs/discussions/categories/q-a) for common issues and solutions.
|
||||
- Join our [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) to ask questions and share your
|
||||
experiences.
|
||||
- Open an issue on our [GitHub Issues](https://github.com/rustfs/rustfs/issues) page for bug reports or feature
|
||||
requests.
|
||||
- Join our [GitHub Discussions](https://github.com/rustfs/rustfs/discussions) to ask questions and share your experiences.
|
||||
- Open an issue on our [GitHub Issues](https://github.com/rustfs/rustfs/issues) page for bug reports or feature requests.
|
||||
|
||||
## Links
|
||||
|
||||
@@ -179,21 +166,21 @@ If you have any questions or need assistance, you can:
|
||||
|
||||
## Contributors
|
||||
|
||||
RustFS is a community-driven project, and we appreciate all contributions. Check out
|
||||
the [Contributors](https://github.com/rustfs/rustfs/graphs/contributors) page to see the amazing people who have helped
|
||||
make RustFS better.
|
||||
RustFS is a community-driven project, and we appreciate all contributions. Check out the [Contributors](https://github.com/rustfs/rustfs/graphs/contributors) page to see the amazing people who have helped make RustFS better.
|
||||
|
||||
<a href="https://github.com/rustfs/rustfs/graphs/contributors">
|
||||
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" alt="Contributors"/>
|
||||
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" />
|
||||
</a>
|
||||
|
||||
|
||||
## Github Trending Top
|
||||
|
||||
🚀 RustFS is beloved by open-source enthusiasts and enterprise users worldwide, often appearing on the GitHub Trending
|
||||
top charts.
|
||||
🚀 RustFS is beloved by open-source enthusiasts and enterprise users worldwide, often appearing on the GitHub Trending top charts.
|
||||
|
||||
<a href="https://trendshift.io/repositories/14181" target="_blank"><img src="https://raw.githubusercontent.com/rustfs/rustfs/refs/heads/main/docs/rustfs-trending.jpg" alt="rustfs%2Frustfs | Trendshift" /></a>
|
||||
|
||||
|
||||
|
||||
## License
|
||||
|
||||
[Apache 2.0](https://opensource.org/licenses/Apache-2.0)
|
||||
|
||||
62
README_ZH.md
62
README_ZH.md
@@ -21,9 +21,7 @@
|
||||
<a href="https://github.com/rustfs/rustfs/blob/main/README.md">English</a > | 简体中文
|
||||
</p >
|
||||
|
||||
RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建的高性能分布式对象存储软件。与 MinIO 一样，它具有简单性、S3
|
||||
兼容性、开源特性以及对数据湖、AI 和大数据的支持等一系列优势。此外，与其他存储系统相比，它采用 Apache
|
||||
许可证构建，拥有更好、更用户友好的开源许可证。由于以 Rust 为基础，RustFS 为高性能对象存储提供了更快的速度和更安全的分布式功能。
|
||||
RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建的高性能分布式对象存储软件。与 MinIO 一样，它具有简单性、S3 兼容性、开源特性以及对数据湖、AI 和大数据的支持等一系列优势。此外，与其他存储系统相比，它采用 Apache 许可证构建，拥有更好、更用户友好的开源许可证。由于以 Rust 为基础，RustFS 为高性能对象存储提供了更快的速度和更安全的分布式功能。
|
||||
|
||||
## 特性
|
||||
|
||||
@@ -38,27 +36,27 @@ RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建
|
||||
|
||||
压力测试服务器参数
|
||||
|
||||
| 类型 | 参数 | 备注 |
|
||||
|-----|----------|----------------------------------------------------------|
|
||||
| CPU | 2 核心 | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz | |
|
||||
| 内存 | 4GB | |
|
||||
| 网络 | 15Gbp | |
|
||||
| 驱动器 | 40GB x 4 | IOPS 3800 / 驱动器 |
|
||||
| 类型 | 参数 | 备注 |
|
||||
| - | - | - |
|
||||
|CPU | 2 核心 | Intel Xeon(Sapphire Rapids) Platinum 8475B , 2.7/3.2 GHz| |
|
||||
|内存| 4GB | |
|
||||
|网络 | 15Gbp | |
|
||||
|驱动器 | 40GB x 4 | IOPS 3800 / 驱动器 |
|
||||
|
||||
<https://github.com/user-attachments/assets/2e4979b5-260c-4f2c-ac12-c87fd558072a>
|
||||
|
||||
### RustFS vs 其他对象存储
|
||||
|
||||
| RustFS | 其他对象存储 |
|
||||
|--------------------------|-------------------------------------|
|
||||
| 强大的控制台 | 简单且无用的控制台 |
|
||||
| 基于 Rust 语言开发，内存更安全 | 使用 Go 或 C 开发，存在内存 GC/泄漏等潜在问题 |
|
||||
| 不向第三方国家报告日志 | 向其他第三方国家报告日志可能违反国家安全法律 |
|
||||
| 采用 Apache 许可证，对商业更友好 | AGPL V3 许可证等其他许可证，污染开源和许可证陷阱，侵犯知识产权 |
|
||||
| 全面的 S3 支持，适用于国内外云提供商 | 完全支持 S3，但不支持本地云厂商 |
|
||||
| 基于 Rust 开发，对安全和创新设备有强大支持 | 对边缘网关和安全创新设备支持较差 |
|
||||
| 稳定的商业价格，免费社区支持 | 高昂的定价，1PiB 成本高达 $250,000 |
|
||||
| 无风险 | 知识产权风险和禁止使用的风险 |
|
||||
| RustFS | 其他对象存储|
|
||||
| - | - |
|
||||
| 强大的控制台 | 简单且无用的控制台 |
|
||||
| 基于 Rust 语言开发，内存更安全 | 使用 Go 或 C 开发，存在内存 GC/泄漏等潜在问题 |
|
||||
| 不向第三方国家报告日志 | 向其他第三方国家报告日志可能违反国家安全法律 |
|
||||
| 采用 Apache 许可证，对商业更友好 | AGPL V3 许可证等其他许可证，污染开源和许可证陷阱，侵犯知识产权 |
|
||||
| 全面的 S3 支持，适用于国内外云提供商 | 完全支持 S3，但不支持本地云厂商 |
|
||||
| 基于 Rust 开发，对安全和创新设备有强大支持 | 对边缘网关和安全创新设备支持较差|
|
||||
| 稳定的商业价格，免费社区支持 | 高昂的定价，1PiB 成本高达 $250,000 |
|
||||
| 无风险 | 知识产权风险和禁止使用的风险 |
|
||||
|
||||
## 快速开始
|
||||
|
||||
@@ -70,30 +68,25 @@ RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建
|
||||
curl -O https://rustfs.com/install_rustfs.sh && bash install_rustfs.sh
|
||||
```
|
||||
|
||||
2. **Docker 快速启动（方案二）**
|
||||
2. **Docker快速启动（方案二）**
|
||||
|
||||
```bash
|
||||
docker run -d -p 9000:9000 -v /data:/data rustfs/rustfs
|
||||
```
|
||||
|
||||
对于使用 Docker 安装来讲，你还可以使用 `docker compose` 来启动 rustfs 实例。在仓库的根目录下面有一个 `docker-compose.yml`
|
||||
文件。运行如下命令即可：
|
||||
对于使用 Docker 安装来讲，你还可以使用 `docker compose` 来启动 rustfs 实例。在仓库的根目录下面有一个 `docker-compose.yml` 文件。运行如下命令即可：
|
||||
|
||||
```
|
||||
docker compose --profile observability up -d
|
||||
```
|
||||
|
||||
**注意**：在使用 `docker compose` 之前，你应该仔细阅读一下 `docker-compose.yaml`，因为该文件中包含多个服务，除了 rustfs 以外，还有 grafana、prometheus、jaeger 等，这些是为 rustfs 可观测性服务的，还有 redis 和 nginx。你想启动哪些容器，就需要用 `--profile` 参数指定相应的 profile。
|
||||
|
||||
**注意**：在使用 `docker compose` 之前，你应该仔细阅读一下 `docker-compose.yaml`，因为该文件中包含多个服务，除了 rustfs
|
||||
以外，还有 grafana、prometheus、jaeger 等，这些是为 rustfs 可观测性服务的，还有 redis 和 nginx。你想启动哪些容器，就需要用
|
||||
`--profile` 参数指定相应的 profile。
|
||||
|
||||
3. **访问控制台**：打开 Web 浏览器并导航到 `http://localhost:9000` 以访问 RustFS 控制台，默认的用户名和密码是
|
||||
`rustfsadmin` 。
|
||||
3. **访问控制台**：打开 Web 浏览器并导航到 `http://localhost:9000` 以访问 RustFS 控制台，默认的用户名和密码是 `rustfsadmin` 。
|
||||
4. **创建存储桶**：使用控制台为您的对象创建新的存储桶。
|
||||
5. **上传对象**：您可以直接通过控制台上传文件，或使用 S3 兼容的 API 与您的 RustFS 实例交互。
|
||||
|
||||
**注意**：如果你想通过 `https` 来访问 RustFS
|
||||
实例，请参考 [TLS 配置文档](https://docs.rustfs.com/zh/integration/tls-configured.html)
|
||||
**注意**：如果你想通过 `https` 来访问 RustFS 实例，请参考 [TLS 配置文档](https://docs.rustfs.com/zh/integration/tls-configured.html)
|
||||
|
||||
## 文档
|
||||
|
||||
@@ -123,19 +116,20 @@ RustFS 是一个使用 Rust（全球最受欢迎的编程语言之一）构建
|
||||
|
||||
## 贡献者
|
||||
|
||||
RustFS 是一个社区驱动的项目，我们感谢所有的贡献。查看[贡献者](https://github.com/rustfs/rustfs/graphs/contributors)页面，了解帮助
|
||||
RustFS 变得更好的杰出人员。
|
||||
RustFS 是一个社区驱动的项目，我们感谢所有的贡献。查看[贡献者](https://github.com/rustfs/rustfs/graphs/contributors)页面，了解帮助 RustFS 变得更好的杰出人员。
|
||||
|
||||
<a href="https://github.com/rustfs/rustfs/graphs/contributors">
|
||||
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" alt="贡献者"/>
|
||||
<img src="https://opencollective.com/rustfs/contributors.svg?width=890&limit=500&button=false" />
|
||||
</a >
|
||||
|
||||
|
||||
## Github 全球推荐榜
|
||||
|
||||
🚀 RustFS 受到了全世界开源爱好者和企业用户的喜欢，多次登顶 Github Trending 全球榜。
|
||||
🚀 RustFS 受到了全世界开源爱好者和企业用户的喜欢，多次登顶Github Trending全球榜。
|
||||
|
||||
<a href="https://trendshift.io/repositories/14181" target="_blank"><img src="https://raw.githubusercontent.com/rustfs/rustfs/refs/heads/main/docs/rustfs-trending.jpg" alt="rustfs%2Frustfs | Trendshift" /></a>
|
||||
|
||||
|
||||
## 许可证
|
||||
|
||||
[Apache 2.0](https://opensource.org/licenses/Apache-2.0)
|
||||
|
||||
@@ -246,7 +246,9 @@ async fn test_performance_impact_measurement() {
|
||||
io_monitor.start().await.unwrap();
|
||||
|
||||
// Baseline test: no scanner load
|
||||
let baseline_duration = measure_workload(5_000, Duration::ZERO).await.max(Duration::from_millis(10));
|
||||
let baseline_start = std::time::Instant::now();
|
||||
simulate_business_workload(1000).await;
|
||||
let baseline_duration = baseline_start.elapsed();
|
||||
|
||||
// Simulate scanner activity
|
||||
scanner.update_business_metrics(50, 500, 0, 25).await;
|
||||
@@ -254,19 +256,13 @@ async fn test_performance_impact_measurement() {
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
// Performance test: with scanner load
|
||||
let with_scanner_duration_raw = measure_workload(5_000, Duration::from_millis(2)).await;
|
||||
let with_scanner_duration = if with_scanner_duration_raw <= baseline_duration {
|
||||
baseline_duration + Duration::from_millis(2)
|
||||
} else {
|
||||
with_scanner_duration_raw
|
||||
};
|
||||
let with_scanner_start = std::time::Instant::now();
|
||||
simulate_business_workload(1000).await;
|
||||
let with_scanner_duration = with_scanner_start.elapsed();
|
||||
|
||||
// Calculate performance impact
|
||||
let baseline_ns = baseline_duration.as_nanos().max(1) as f64;
|
||||
let overhead_duration = with_scanner_duration.saturating_sub(baseline_duration);
|
||||
let overhead_ns = overhead_duration.as_nanos() as f64;
|
||||
let overhead_ms = (overhead_ns / 1_000_000.0).round() as u64;
|
||||
let impact_percentage = (overhead_ns / baseline_ns) * 100.0;
|
||||
let overhead_ms = with_scanner_duration.saturating_sub(baseline_duration).as_millis() as u64;
|
||||
let impact_percentage = (overhead_ms as f64 / baseline_duration.as_millis() as f64) * 100.0;
|
||||
|
||||
let benchmark = PerformanceBenchmark {
|
||||
_scanner_overhead_ms: overhead_ms,
|
||||
@@ -361,15 +357,6 @@ async fn simulate_business_workload(operations: usize) {
|
||||
}
|
||||
}
|
||||
|
||||
async fn measure_workload(operations: usize, extra_delay: Duration) -> Duration {
|
||||
let start = std::time::Instant::now();
|
||||
simulate_business_workload(operations).await;
|
||||
if !extra_delay.is_zero() {
|
||||
tokio::time::sleep(extra_delay).await;
|
||||
}
|
||||
start.elapsed()
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_error_recovery_and_resilience() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
|
||||
@@ -343,7 +343,7 @@ mod serial_tests {
|
||||
set_bucket_lifecycle(bucket_name.as_str())
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
|
||||
println!("✅ Lifecycle configuration set for bucket: {}", bucket_name);
|
||||
|
||||
// Verify lifecycle configuration was set
|
||||
match rustfs_ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
|
||||
@@ -477,7 +477,7 @@ mod serial_tests {
|
||||
set_bucket_lifecycle_deletemarker(bucket_name.as_str())
|
||||
.await
|
||||
.expect("Failed to set lifecycle configuration");
|
||||
println!("✅ Lifecycle configuration set for bucket: {bucket_name}");
|
||||
println!("✅ Lifecycle configuration set for bucket: {}", bucket_name);
|
||||
|
||||
// Verify lifecycle configuration was set
|
||||
match rustfs_ecstore::bucket::metadata_sys::get(bucket_name.as_str()).await {
|
||||
|
||||
@@ -37,6 +37,7 @@ thiserror = { workspace = true }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
|
||||
tracing = { workspace = true, features = ["std", "attributes"] }
|
||||
url = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
rumqttc = { workspace = true }
|
||||
|
||||
[lints]
|
||||
|
||||
@@ -13,12 +13,13 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{AuditEntry, AuditResult, AuditSystem};
|
||||
use once_cell::sync::OnceCell;
|
||||
use rustfs_ecstore::config::Config;
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::sync::Arc;
|
||||
use tracing::{error, warn};
|
||||
|
||||
/// Global audit system instance
|
||||
static AUDIT_SYSTEM: OnceLock<Arc<AuditSystem>> = OnceLock::new();
|
||||
static AUDIT_SYSTEM: OnceCell<Arc<AuditSystem>> = OnceCell::new();
|
||||
|
||||
/// Initialize the global audit system
|
||||
pub fn init_audit_system() -> Arc<AuditSystem> {
|
||||
|
||||
@@ -21,8 +21,8 @@
|
||||
//! - Error rate monitoring
|
||||
//! - Queue depth monitoring
|
||||
|
||||
use std::sync::Arc;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::time::{Duration, Instant};
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::info;
|
||||
@@ -312,7 +312,7 @@ impl PerformanceValidation {
|
||||
}
|
||||
|
||||
/// Global metrics instance
|
||||
static GLOBAL_METRICS: OnceLock<Arc<AuditMetrics>> = OnceLock::new();
|
||||
static GLOBAL_METRICS: once_cell::sync::OnceCell<Arc<AuditMetrics>> = once_cell::sync::OnceCell::new();
|
||||
|
||||
/// Get or initialize the global metrics instance
|
||||
pub fn global_metrics() -> Arc<AuditMetrics> {
|
||||
|
||||
@@ -12,19 +12,20 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{AuditEntry, AuditError, AuditResult};
|
||||
use futures::{StreamExt, stream::FuturesUnordered};
|
||||
use crate::AuditEntry;
|
||||
use crate::{AuditError, AuditResult};
|
||||
use futures::StreamExt;
|
||||
use futures::stream::FuturesUnordered;
|
||||
use rustfs_config::audit::AUDIT_ROUTE_PREFIX;
|
||||
use rustfs_config::{
|
||||
DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, MQTT_BROKER, MQTT_KEEP_ALIVE_INTERVAL, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR,
|
||||
MQTT_QUEUE_LIMIT, MQTT_RECONNECT_INTERVAL, MQTT_TOPIC, MQTT_USERNAME, WEBHOOK_AUTH_TOKEN, WEBHOOK_BATCH_SIZE,
|
||||
WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_HTTP_TIMEOUT, WEBHOOK_MAX_RETRY, WEBHOOK_QUEUE_DIR,
|
||||
WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL, audit::AUDIT_ROUTE_PREFIX,
|
||||
WEBHOOK_QUEUE_LIMIT, WEBHOOK_RETRY_INTERVAL,
|
||||
};
|
||||
use rustfs_ecstore::config::{Config, KVS};
|
||||
use rustfs_targets::{
|
||||
Target, TargetError,
|
||||
target::{ChannelTargetType, TargetType, mqtt::MQTTArgs, webhook::WebhookArgs},
|
||||
};
|
||||
use rustfs_targets::target::{ChannelTargetType, TargetType, mqtt::MQTTArgs, webhook::WebhookArgs};
|
||||
use rustfs_targets::{Target, TargetError};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
@@ -67,10 +68,7 @@ impl AuditRegistry {
|
||||
|
||||
// A collection of asynchronous tasks for concurrently executing target creation
|
||||
let mut tasks = FuturesUnordered::new();
|
||||
// let final_config = config.clone();
|
||||
|
||||
// Record the defaults for each segment so that the segment can eventually be rebuilt
|
||||
let mut section_defaults: HashMap<String, KVS> = HashMap::new();
|
||||
let mut final_config = config.clone();
|
||||
|
||||
// Supported target types for audit
|
||||
let target_types = vec![ChannelTargetType::Webhook.as_str(), ChannelTargetType::Mqtt.as_str()];
|
||||
@@ -82,14 +80,11 @@ impl AuditRegistry {
|
||||
info!(target_type = %target_type, "Starting audit target type processing");
|
||||
|
||||
// 2. Prepare the configuration source
|
||||
let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
|
||||
let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}");
|
||||
let file_configs = config.0.get(§ion_name).cloned().unwrap_or_default();
|
||||
let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
|
||||
debug!(?default_cfg, "Retrieved default configuration");
|
||||
|
||||
// Save defaults for eventual write back
|
||||
section_defaults.insert(section_name.clone(), default_cfg.clone());
|
||||
|
||||
// Get valid fields for the target type
|
||||
let valid_fields = match target_type {
|
||||
"webhook" => get_webhook_valid_fields(),
|
||||
@@ -106,7 +101,7 @@ impl AuditRegistry {
|
||||
let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();
|
||||
|
||||
for (env_key, env_value) in &all_env {
|
||||
let audit_prefix = format!("{ENV_PREFIX}{AUDIT_ROUTE_PREFIX}{target_type}").to_uppercase();
|
||||
let audit_prefix = format!("{ENV_PREFIX}AUDIT_{}", target_type.to_uppercase());
|
||||
if !env_key.starts_with(&audit_prefix) {
|
||||
continue;
|
||||
}
|
||||
@@ -191,33 +186,38 @@ impl AuditRegistry {
|
||||
let target_type_clone = target_type.to_string();
|
||||
let id_clone = id.clone();
|
||||
let merged_config_arc = Arc::new(merged_config.clone());
|
||||
let final_config_arc = Arc::new(final_config.clone());
|
||||
|
||||
let task = tokio::spawn(async move {
|
||||
let result = create_audit_target(&target_type_clone, &id_clone, &merged_config_arc).await;
|
||||
(target_type_clone, id_clone, result, merged_config_arc)
|
||||
(target_type_clone, id_clone, result, final_config_arc)
|
||||
});
|
||||
|
||||
tasks.push(task);
|
||||
|
||||
// Update final config with successful instance
|
||||
// final_config.0.entry(section_name.clone()).or_default().insert(id, merged_config);
|
||||
final_config
|
||||
.0
|
||||
.entry(section_name.clone())
|
||||
.or_default()
|
||||
.insert(id, merged_config);
|
||||
} else {
|
||||
info!(instance_id = %id, "Skipping disabled audit target, will be removed from final configuration");
|
||||
// Remove disabled target from final configuration
|
||||
// final_config.0.entry(section_name.clone()).or_default().remove(&id);
|
||||
final_config.0.entry(section_name.clone()).or_default().remove(&id);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Concurrently execute all creation tasks and collect results
|
||||
let mut successful_targets = Vec::new();
|
||||
let mut successful_configs = Vec::new();
|
||||
|
||||
while let Some(task_result) = tasks.next().await {
|
||||
match task_result {
|
||||
Ok((target_type, id, result, kvs_arc)) => match result {
|
||||
Ok((target_type, id, result, _final_config)) => match result {
|
||||
Ok(target) => {
|
||||
info!(target_type = %target_type, instance_id = %id, "Created audit target successfully");
|
||||
successful_targets.push(target);
|
||||
successful_configs.push((target_type, id, kvs_arc));
|
||||
}
|
||||
Err(e) => {
|
||||
error!(target_type = %target_type, instance_id = %id, error = %e, "Failed to create audit target");
|
||||
@@ -229,67 +229,21 @@ impl AuditRegistry {
|
||||
}
|
||||
}
|
||||
|
||||
// Rebuild in pieces based on "default items + successful instances" and overwrite writeback to ensure that deleted/disabled instances will not be "resurrected"
|
||||
if !successful_configs.is_empty() || !section_defaults.is_empty() {
|
||||
info!("Prepare to rebuild and save target configurations to the system configuration...");
|
||||
// 7. Save the new configuration to the system
|
||||
let Some(store) = rustfs_ecstore::new_object_layer_fn() else {
|
||||
return Err(AuditError::ServerNotInitialized(
|
||||
"Failed to save target configuration: server storage not initialized".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
// Aggregate successful instances into segments
|
||||
let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
|
||||
for (target_type, id, kvs) in successful_configs {
|
||||
let section_name = format!("{AUDIT_ROUTE_PREFIX}{target_type}").to_lowercase();
|
||||
successes_by_section
|
||||
.entry(section_name)
|
||||
.or_default()
|
||||
.insert(id.to_lowercase(), (*kvs).clone());
|
||||
}
|
||||
|
||||
let mut new_config = config.clone();
|
||||
|
||||
// Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
|
||||
let mut sections: HashSet<String> = HashSet::new();
|
||||
sections.extend(section_defaults.keys().cloned());
|
||||
sections.extend(successes_by_section.keys().cloned());
|
||||
|
||||
for section_name in sections {
|
||||
let mut section_map: HashMap<String, KVS> = HashMap::new();
|
||||
|
||||
// The default entry (if present) is written back to `_`
|
||||
if let Some(default_cfg) = section_defaults.get(§ion_name) {
|
||||
if !default_cfg.is_empty() {
|
||||
section_map.insert(DEFAULT_DELIMITER.to_string(), default_cfg.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Successful instance write back
|
||||
if let Some(instances) = successes_by_section.get(§ion_name) {
|
||||
for (id, kvs) in instances {
|
||||
section_map.insert(id.clone(), kvs.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Empty segments are removed and non-empty segments are replaced as a whole.
|
||||
if section_map.is_empty() {
|
||||
new_config.0.remove(§ion_name);
|
||||
} else {
|
||||
new_config.0.insert(section_name, section_map);
|
||||
}
|
||||
}
|
||||
|
||||
// 7. Save the new configuration to the system
|
||||
let Some(store) = rustfs_ecstore::new_object_layer_fn() else {
|
||||
return Err(AuditError::ServerNotInitialized(
|
||||
"Failed to save target configuration: server storage not initialized".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
match rustfs_ecstore::config::com::save_server_config(store, &new_config).await {
|
||||
Ok(_) => info!("New audit configuration saved to system successfully"),
|
||||
Err(e) => {
|
||||
error!(error = %e, "Failed to save new audit configuration");
|
||||
return Err(AuditError::SaveConfig(e.to_string()));
|
||||
}
|
||||
match rustfs_ecstore::config::com::save_server_config(store, &final_config).await {
|
||||
Ok(_) => info!("New audit configuration saved to system successfully"),
|
||||
Err(e) => {
|
||||
error!(error = %e, "Failed to save new audit configuration");
|
||||
return Err(AuditError::SaveConfig(e.to_string()));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(successful_targets)
|
||||
}
|
||||
|
||||
|
||||
@@ -12,7 +12,10 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{AuditEntry, AuditError, AuditRegistry, AuditResult, observability};
|
||||
use crate::AuditEntry;
|
||||
use crate::AuditRegistry;
|
||||
use crate::observability;
|
||||
use crate::{AuditError, AuditResult};
|
||||
use rustfs_ecstore::config::Config;
|
||||
use rustfs_targets::{
|
||||
StoreError, Target, TargetError,
|
||||
|
||||
@@ -81,8 +81,8 @@ fn test_config_section_names() {
|
||||
fn test_environment_variable_parsing() {
|
||||
// Test environment variable prefix patterns
|
||||
let env_prefix = "RUSTFS_";
|
||||
let audit_webhook_prefix = format!("{env_prefix}AUDIT_WEBHOOK_");
|
||||
let audit_mqtt_prefix = format!("{env_prefix}AUDIT_MQTT_");
|
||||
let audit_webhook_prefix = format!("{}AUDIT_WEBHOOK_", env_prefix);
|
||||
let audit_mqtt_prefix = format!("{}AUDIT_MQTT_", env_prefix);
|
||||
|
||||
assert_eq!(audit_webhook_prefix, "RUSTFS_AUDIT_WEBHOOK_");
|
||||
assert_eq!(audit_mqtt_prefix, "RUSTFS_AUDIT_MQTT_");
|
||||
@@ -141,13 +141,13 @@ fn test_duration_parsing_formats() {
|
||||
let result = parse_duration_test(input);
|
||||
match (result, expected_seconds) {
|
||||
(Some(duration), Some(expected)) => {
|
||||
assert_eq!(duration.as_secs(), expected, "Failed for input: {input}");
|
||||
assert_eq!(duration.as_secs(), expected, "Failed for input: {}", input);
|
||||
}
|
||||
(None, None) => {
|
||||
// Both None, test passes
|
||||
}
|
||||
_ => {
|
||||
panic!("Mismatch for input: {input}, got: {result:?}, expected: {expected_seconds:?}");
|
||||
panic!("Mismatch for input: {}, got: {:?}, expected: {:?}", input, result, expected_seconds);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -188,13 +188,13 @@ fn test_url_validation() {
|
||||
|
||||
for url_str in valid_urls {
|
||||
let result = Url::parse(url_str);
|
||||
assert!(result.is_ok(), "Valid URL should parse: {url_str}");
|
||||
assert!(result.is_ok(), "Valid URL should parse: {}", url_str);
|
||||
}
|
||||
|
||||
for url_str in &invalid_urls[..3] {
|
||||
// Skip the ftp one as it's technically valid
|
||||
let result = Url::parse(url_str);
|
||||
assert!(result.is_err(), "Invalid URL should not parse: {url_str}");
|
||||
assert!(result.is_err(), "Invalid URL should not parse: {}", url_str);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -214,6 +214,6 @@ fn test_qos_parsing() {
|
||||
0..=2 => Some(q),
|
||||
_ => None,
|
||||
});
|
||||
assert_eq!(result, expected, "Failed for QoS input: {input}");
|
||||
assert_eq!(result, expected, "Failed for QoS input: {}", input);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -57,7 +57,7 @@ async fn test_config_parsing_webhook() {
|
||||
}
|
||||
Err(e) => {
|
||||
// Other errors might indicate parsing issues
|
||||
println!("Unexpected error: {e}");
|
||||
println!("Unexpected error: {}", e);
|
||||
}
|
||||
Ok(_) => {
|
||||
// Unexpected success in test environment without server storage
|
||||
@@ -103,6 +103,6 @@ fn test_enable_value_parsing() {
|
||||
|
||||
for (input, expected) in test_cases {
|
||||
let result = matches!(input.to_lowercase().as_str(), "1" | "on" | "true" | "yes");
|
||||
assert_eq!(result, expected, "Failed for input: {input}");
|
||||
assert_eq!(result, expected, "Failed for input: {}", input);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,10 +32,10 @@ async fn test_audit_system_startup_performance() {
|
||||
let _result = timeout(Duration::from_secs(5), system.start(config)).await;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
println!("Audit system startup took: {elapsed:?}");
|
||||
println!("Audit system startup took: {:?}", elapsed);
|
||||
|
||||
// Should complete within 5 seconds
|
||||
assert!(elapsed < Duration::from_secs(5), "Startup took too long: {elapsed:?}");
|
||||
assert!(elapsed < Duration::from_secs(5), "Startup took too long: {:?}", elapsed);
|
||||
|
||||
// Clean up
|
||||
let _ = system.close().await;
|
||||
@@ -54,8 +54,8 @@ async fn test_concurrent_target_creation() {
|
||||
for i in 1..=5 {
|
||||
let mut kvs = rustfs_ecstore::config::KVS::new();
|
||||
kvs.insert("enable".to_string(), "on".to_string());
|
||||
kvs.insert("endpoint".to_string(), format!("http://localhost:302{i}/webhook"));
|
||||
webhook_section.insert(format!("instance_{i}"), kvs);
|
||||
kvs.insert("endpoint".to_string(), format!("http://localhost:302{}/webhook", i));
|
||||
webhook_section.insert(format!("instance_{}", i), kvs);
|
||||
}
|
||||
|
||||
config.0.insert("audit_webhook".to_string(), webhook_section);
|
||||
@@ -66,10 +66,10 @@ async fn test_concurrent_target_creation() {
|
||||
let result = registry.create_targets_from_config(&config).await;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
println!("Concurrent target creation took: {elapsed:?}");
|
||||
println!("Concurrent target creation took: {:?}", elapsed);
|
||||
|
||||
// Should complete quickly even with multiple targets
|
||||
assert!(elapsed < Duration::from_secs(10), "Target creation took too long: {elapsed:?}");
|
||||
assert!(elapsed < Duration::from_secs(10), "Target creation took too long: {:?}", elapsed);
|
||||
|
||||
// Verify it fails with expected error (server not initialized)
|
||||
match result {
|
||||
@@ -77,7 +77,7 @@ async fn test_concurrent_target_creation() {
|
||||
// Expected in test environment
|
||||
}
|
||||
Err(e) => {
|
||||
println!("Unexpected error during concurrent creation: {e}");
|
||||
println!("Unexpected error during concurrent creation: {}", e);
|
||||
}
|
||||
Ok(_) => {
|
||||
println!("Unexpected success in test environment");
|
||||
@@ -93,7 +93,7 @@ async fn test_audit_log_dispatch_performance() {
|
||||
let config = rustfs_ecstore::config::Config(HashMap::new());
|
||||
let start_result = system.start(config).await;
|
||||
if start_result.is_err() {
|
||||
println!("AuditSystem failed to start: {start_result:?}");
|
||||
println!("AuditSystem failed to start: {:?}", start_result);
|
||||
return; // 或 assert!(false, "AuditSystem failed to start");
|
||||
}
|
||||
|
||||
@@ -104,14 +104,14 @@ async fn test_audit_log_dispatch_performance() {
|
||||
let id = 1;
|
||||
|
||||
let mut req_header = HashMap::new();
|
||||
req_header.insert("authorization".to_string(), format!("Bearer test-token-{id}"));
|
||||
req_header.insert("authorization".to_string(), format!("Bearer test-token-{}", id));
|
||||
req_header.insert("content-type".to_string(), "application/octet-stream".to_string());
|
||||
|
||||
let mut resp_header = HashMap::new();
|
||||
resp_header.insert("x-response".to_string(), "ok".to_string());
|
||||
|
||||
let mut tags = HashMap::new();
|
||||
tags.insert(format!("tag-{id}"), json!("sample"));
|
||||
tags.insert(format!("tag-{}", id), json!("sample"));
|
||||
|
||||
let mut req_query = HashMap::new();
|
||||
req_query.insert("id".to_string(), id.to_string());
|
||||
@@ -119,7 +119,7 @@ async fn test_audit_log_dispatch_performance() {
|
||||
let api_details = ApiDetails {
|
||||
name: Some("PutObject".to_string()),
|
||||
bucket: Some("test-bucket".to_string()),
|
||||
object: Some(format!("test-object-{id}")),
|
||||
object: Some(format!("test-object-{}", id)),
|
||||
status: Some("success".to_string()),
|
||||
status_code: Some(200),
|
||||
input_bytes: Some(1024),
|
||||
@@ -134,7 +134,7 @@ async fn test_audit_log_dispatch_performance() {
|
||||
// Create sample audit log entry
|
||||
let audit_entry = AuditEntry {
|
||||
version: "1".to_string(),
|
||||
deployment_id: Some(format!("test-deployment-{id}")),
|
||||
deployment_id: Some(format!("test-deployment-{}", id)),
|
||||
site_name: Some("test-site".to_string()),
|
||||
time: Utc::now(),
|
||||
event: EventName::ObjectCreatedPut,
|
||||
@@ -142,9 +142,9 @@ async fn test_audit_log_dispatch_performance() {
|
||||
trigger: "api".to_string(),
|
||||
api: api_details,
|
||||
remote_host: Some("127.0.0.1".to_string()),
|
||||
request_id: Some(format!("test-request-{id}")),
|
||||
request_id: Some(format!("test-request-{}", id)),
|
||||
user_agent: Some("test-agent".to_string()),
|
||||
req_path: Some(format!("/test-bucket/test-object-{id}")),
|
||||
req_path: Some(format!("/test-bucket/test-object-{}", id)),
|
||||
req_host: Some("test-host".to_string()),
|
||||
req_node: Some("node-1".to_string()),
|
||||
req_claims: None,
|
||||
@@ -152,8 +152,8 @@ async fn test_audit_log_dispatch_performance() {
|
||||
req_header: Some(req_header),
|
||||
resp_header: Some(resp_header),
|
||||
tags: Some(tags),
|
||||
access_key: Some(format!("AKIA{id}")),
|
||||
parent_user: Some(format!("parent-{id}")),
|
||||
access_key: Some(format!("AKIA{}", id)),
|
||||
parent_user: Some(format!("parent-{}", id)),
|
||||
error: None,
|
||||
};
|
||||
|
||||
@@ -163,10 +163,10 @@ async fn test_audit_log_dispatch_performance() {
|
||||
let result = system.dispatch(Arc::new(audit_entry)).await;
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
println!("Audit log dispatch took: {elapsed:?}");
|
||||
println!("Audit log dispatch took: {:?}", elapsed);
|
||||
|
||||
// Should be very fast (sub-millisecond for no targets)
|
||||
assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {elapsed:?}");
|
||||
assert!(elapsed < Duration::from_millis(100), "Dispatch took too long: {:?}", elapsed);
|
||||
|
||||
// Should succeed even with no targets
|
||||
assert!(result.is_ok(), "Dispatch should succeed with no targets");
|
||||
@@ -226,10 +226,10 @@ fn test_event_name_mask_performance() {
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
println!("Event mask calculation (5000 ops) took: {elapsed:?}");
|
||||
println!("Event mask calculation (5000 ops) took: {:?}", elapsed);
|
||||
|
||||
// Should be very fast
|
||||
assert!(elapsed < Duration::from_millis(100), "Mask calculation too slow: {elapsed:?}");
|
||||
assert!(elapsed < Duration::from_millis(100), "Mask calculation too slow: {:?}", elapsed);
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -254,10 +254,10 @@ fn test_event_name_expansion_performance() {
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
println!("Event expansion (4000 ops) took: {elapsed:?}");
|
||||
println!("Event expansion (4000 ops) took: {:?}", elapsed);
|
||||
|
||||
// Should be very fast
|
||||
assert!(elapsed < Duration::from_millis(100), "Expansion too slow: {elapsed:?}");
|
||||
assert!(elapsed < Duration::from_millis(100), "Expansion too slow: {:?}", elapsed);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
@@ -274,10 +274,10 @@ async fn test_registry_operations_performance() {
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
println!("Registry operations (2000 ops) took: {elapsed:?}");
|
||||
println!("Registry operations (2000 ops) took: {:?}", elapsed);
|
||||
|
||||
// Should be very fast for empty registry
|
||||
assert!(elapsed < Duration::from_millis(100), "Registry ops too slow: {elapsed:?}");
|
||||
assert!(elapsed < Duration::from_millis(100), "Registry ops too slow: {:?}", elapsed);
|
||||
}
|
||||
|
||||
// Performance requirements validation
|
||||
@@ -294,7 +294,7 @@ fn test_performance_requirements() {
|
||||
// Simulate processing 3000 events worth of operations
|
||||
for i in 0..3000 {
|
||||
// Simulate event name parsing and processing
|
||||
let _event_id = format!("s3:ObjectCreated:Put_{i}");
|
||||
let _event_id = format!("s3:ObjectCreated:Put_{}", i);
|
||||
let _timestamp = chrono::Utc::now().to_rfc3339();
|
||||
|
||||
// Simulate basic audit entry creation overhead
|
||||
@@ -305,16 +305,16 @@ fn test_performance_requirements() {
|
||||
let elapsed = start.elapsed();
|
||||
let eps = 3000.0 / elapsed.as_secs_f64();
|
||||
|
||||
println!("Simulated 3000 events in {elapsed:?} ({eps:.0} EPS)");
|
||||
println!("Simulated 3000 events in {:?} ({:.0} EPS)", elapsed, eps);
|
||||
|
||||
// Our core processing should easily handle 3k EPS worth of CPU overhead
|
||||
// The actual EPS limit will be determined by network I/O to targets
|
||||
assert!(eps > 10000.0, "Core processing too slow for 3k EPS target: {eps} EPS");
|
||||
assert!(eps > 10000.0, "Core processing too slow for 3k EPS target: {} EPS", eps);
|
||||
|
||||
// P99 latency requirement: < 30ms
|
||||
// For core processing, we should be much faster than this
|
||||
let avg_latency = elapsed / 3000;
|
||||
println!("Average processing latency: {avg_latency:?}");
|
||||
println!("Average processing latency: {:?}", avg_latency);
|
||||
|
||||
assert!(avg_latency < Duration::from_millis(1), "Processing latency too high: {avg_latency:?}");
|
||||
assert!(avg_latency < Duration::from_millis(1), "Processing latency too high: {:?}", avg_latency);
|
||||
}
|
||||
|
||||
@@ -52,7 +52,7 @@ async fn test_complete_audit_system_lifecycle() {
|
||||
assert_eq!(system.get_state().await, system::AuditSystemState::Running);
|
||||
}
|
||||
Err(e) => {
|
||||
panic!("Unexpected error: {e}");
|
||||
panic!("Unexpected error: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -103,7 +103,7 @@ async fn test_audit_log_dispatch_with_no_targets() {
|
||||
// Also acceptable since system not running
|
||||
}
|
||||
Err(e) => {
|
||||
panic!("Unexpected error: {e}");
|
||||
panic!("Unexpected error: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -172,7 +172,7 @@ async fn test_config_parsing_with_multiple_instances() {
|
||||
// Expected - parsing worked but save failed
|
||||
}
|
||||
Err(e) => {
|
||||
println!("Config parsing error: {e}");
|
||||
println!("Config parsing error: {}", e);
|
||||
// Other errors might indicate parsing issues, but not necessarily failures
|
||||
}
|
||||
Ok(_) => {
|
||||
@@ -261,7 +261,7 @@ async fn test_concurrent_operations() {
|
||||
let (i, state, is_running) = task.await.expect("Task should complete");
|
||||
assert_eq!(state, system::AuditSystemState::Stopped);
|
||||
assert!(!is_running);
|
||||
println!("Task {i} completed successfully");
|
||||
println!("Task {} completed successfully", i);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -295,8 +295,8 @@ async fn test_performance_under_load() {
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
println!("100 concurrent dispatches took: {elapsed:?}");
|
||||
println!("Successes: {success_count}, Errors: {error_count}");
|
||||
println!("100 concurrent dispatches took: {:?}", elapsed);
|
||||
println!("Successes: {}, Errors: {}", success_count, error_count);
|
||||
|
||||
// Should complete reasonably quickly
|
||||
assert!(elapsed < Duration::from_secs(5), "Concurrent operations took too long");
|
||||
@@ -318,14 +318,14 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
|
||||
use std::collections::HashMap;
|
||||
|
||||
let mut req_header = HashMap::new();
|
||||
req_header.insert("authorization".to_string(), format!("Bearer test-token-{id}"));
|
||||
req_header.insert("authorization".to_string(), format!("Bearer test-token-{}", id));
|
||||
req_header.insert("content-type".to_string(), "application/octet-stream".to_string());
|
||||
|
||||
let mut resp_header = HashMap::new();
|
||||
resp_header.insert("x-response".to_string(), "ok".to_string());
|
||||
|
||||
let mut tags = HashMap::new();
|
||||
tags.insert(format!("tag-{id}"), json!("sample"));
|
||||
tags.insert(format!("tag-{}", id), json!("sample"));
|
||||
|
||||
let mut req_query = HashMap::new();
|
||||
req_query.insert("id".to_string(), id.to_string());
|
||||
@@ -333,7 +333,7 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
|
||||
let api_details = ApiDetails {
|
||||
name: Some("PutObject".to_string()),
|
||||
bucket: Some("test-bucket".to_string()),
|
||||
object: Some(format!("test-object-{id}")),
|
||||
object: Some(format!("test-object-{}", id)),
|
||||
status: Some("success".to_string()),
|
||||
status_code: Some(200),
|
||||
input_bytes: Some(1024),
|
||||
@@ -348,7 +348,7 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
|
||||
|
||||
AuditEntry {
|
||||
version: "1".to_string(),
|
||||
deployment_id: Some(format!("test-deployment-{id}")),
|
||||
deployment_id: Some(format!("test-deployment-{}", id)),
|
||||
site_name: Some("test-site".to_string()),
|
||||
time: Utc::now(),
|
||||
event: EventName::ObjectCreatedPut,
|
||||
@@ -356,9 +356,9 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
|
||||
trigger: "api".to_string(),
|
||||
api: api_details,
|
||||
remote_host: Some("127.0.0.1".to_string()),
|
||||
request_id: Some(format!("test-request-{id}")),
|
||||
request_id: Some(format!("test-request-{}", id)),
|
||||
user_agent: Some("test-agent".to_string()),
|
||||
req_path: Some(format!("/test-bucket/test-object-{id}")),
|
||||
req_path: Some(format!("/test-bucket/test-object-{}", id)),
|
||||
req_host: Some("test-host".to_string()),
|
||||
req_node: Some("node-1".to_string()),
|
||||
req_claims: None,
|
||||
@@ -366,8 +366,8 @@ fn create_sample_audit_entry_with_id(id: u32) -> AuditEntry {
|
||||
req_header: Some(req_header),
|
||||
resp_header: Some(resp_header),
|
||||
tags: Some(tags),
|
||||
access_key: Some(format!("AKIA{id}")),
|
||||
parent_user: Some(format!("parent-{id}")),
|
||||
access_key: Some(format!("AKIA{}", id)),
|
||||
parent_user: Some(format!("parent-{}", id)),
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -36,4 +36,4 @@ audit = ["dep:const-str", "constants"]
|
||||
constants = ["dep:const-str"]
|
||||
notify = ["dep:const-str", "constants"]
|
||||
observability = ["constants"]
|
||||
opa = ["constants"]
|
||||
|
||||
|
||||
@@ -126,6 +126,12 @@ pub const DEFAULT_LOG_FILENAME: &str = "rustfs";
|
||||
/// Default value: rustfs.log
|
||||
pub const DEFAULT_OBS_LOG_FILENAME: &str = concat!(DEFAULT_LOG_FILENAME, "");
|
||||
|
||||
/// Default sink file log file for rustfs
|
||||
/// This is the default sink file log file for rustfs.
|
||||
/// It is used to store the logs of the application.
|
||||
/// Default value: rustfs-sink.log
|
||||
pub const DEFAULT_SINK_FILE_LOG_FILE: &str = concat!(DEFAULT_LOG_FILENAME, "-sink.log");
|
||||
|
||||
/// Default log directory for rustfs
|
||||
/// This is the default log directory for rustfs.
|
||||
/// It is used to store the logs of the application.
|
||||
@@ -154,6 +160,16 @@ pub const DEFAULT_LOG_ROTATION_TIME: &str = "day";
|
||||
/// Environment variable: RUSTFS_OBS_LOG_KEEP_FILES
|
||||
pub const DEFAULT_LOG_KEEP_FILES: u16 = 30;
|
||||
|
||||
/// This is the external address for rustfs to access endpoint (used in Docker deployments).
|
||||
/// This should match the mapped host port when using Docker port mapping.
|
||||
/// Example: ":9020" when mapping host port 9020 to container port 9000.
|
||||
/// Default value: DEFAULT_ADDRESS
|
||||
/// Environment variable: RUSTFS_EXTERNAL_ADDRESS
|
||||
/// Command line argument: --external-address
|
||||
/// Example: RUSTFS_EXTERNAL_ADDRESS=":9020"
|
||||
/// Example: --external-address ":9020"
|
||||
pub const ENV_EXTERNAL_ADDRESS: &str = "RUSTFS_EXTERNAL_ADDRESS";
|
||||
|
||||
/// 1 KiB
|
||||
pub const KI_B: usize = 1024;
|
||||
/// 1 MiB
|
||||
|
||||
@@ -32,5 +32,3 @@ pub mod audit;
|
||||
pub mod notify;
|
||||
#[cfg(feature = "observability")]
|
||||
pub mod observability;
|
||||
#[cfg(feature = "opa")]
|
||||
pub mod opa;
|
||||
|
||||
98
crates/config/src/observability/config.rs
Normal file
98
crates/config/src/observability/config.rs
Normal file
@@ -0,0 +1,98 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Observability Keys
|
||||
|
||||
pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
|
||||
pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
|
||||
pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
|
||||
pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
|
||||
pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
|
||||
pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
|
||||
pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOCAL_LOGGING_ENABLED: &str = "RUSTFS_OBS_LOCAL_LOGGING_ENABLED";
|
||||
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
||||
pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
|
||||
pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB";
|
||||
pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
|
||||
pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
|
||||
|
||||
/// Log pool capacity for async logging
|
||||
pub const ENV_OBS_LOG_POOL_CAPA: &str = "RUSTFS_OBS_LOG_POOL_CAPA";
|
||||
|
||||
/// Log message capacity for async logging
|
||||
pub const ENV_OBS_LOG_MESSAGE_CAPA: &str = "RUSTFS_OBS_LOG_MESSAGE_CAPA";
|
||||
|
||||
/// Log flush interval in milliseconds for async logging
|
||||
pub const ENV_OBS_LOG_FLUSH_MS: &str = "RUSTFS_OBS_LOG_FLUSH_MS";
|
||||
|
||||
/// Default values for log pool
|
||||
pub const DEFAULT_OBS_LOG_POOL_CAPA: usize = 10240;
|
||||
|
||||
/// Default values for message capacity
|
||||
pub const DEFAULT_OBS_LOG_MESSAGE_CAPA: usize = 32768;
|
||||
|
||||
/// Default values for flush interval in milliseconds
|
||||
pub const DEFAULT_OBS_LOG_FLUSH_MS: u64 = 200;
|
||||
|
||||
/// Audit logger queue capacity environment variable key
|
||||
pub const ENV_AUDIT_LOGGER_QUEUE_CAPACITY: &str = "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY";
|
||||
|
||||
/// Default values for observability configuration
|
||||
pub const DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY: usize = 10000;
|
||||
|
||||
/// Default values for observability configuration
|
||||
// ### Supported Environment Values
|
||||
// - `production` - Secure file-only logging
|
||||
// - `development` - Full debugging with stdout
|
||||
// - `test` - Test environment with stdout support
|
||||
// - `staging` - Staging environment with stdout support
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_PRODUCTION: &str = "production";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT: &str = "development";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_TEST: &str = "test";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_STAGING: &str = "staging";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_env_keys() {
|
||||
assert_eq!(ENV_OBS_ENDPOINT, "RUSTFS_OBS_ENDPOINT");
|
||||
assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
|
||||
assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");
|
||||
assert_eq!(ENV_OBS_METER_INTERVAL, "RUSTFS_OBS_METER_INTERVAL");
|
||||
assert_eq!(ENV_OBS_SERVICE_NAME, "RUSTFS_OBS_SERVICE_NAME");
|
||||
assert_eq!(ENV_OBS_SERVICE_VERSION, "RUSTFS_OBS_SERVICE_VERSION");
|
||||
assert_eq!(ENV_OBS_ENVIRONMENT, "RUSTFS_OBS_ENVIRONMENT");
|
||||
assert_eq!(ENV_OBS_LOGGER_LEVEL, "RUSTFS_OBS_LOGGER_LEVEL");
|
||||
assert_eq!(ENV_OBS_LOCAL_LOGGING_ENABLED, "RUSTFS_OBS_LOCAL_LOGGING_ENABLED");
|
||||
assert_eq!(ENV_OBS_LOG_DIRECTORY, "RUSTFS_OBS_LOG_DIRECTORY");
|
||||
assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
|
||||
assert_eq!(ENV_OBS_LOG_ROTATION_SIZE_MB, "RUSTFS_OBS_LOG_ROTATION_SIZE_MB");
|
||||
assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
|
||||
assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
|
||||
assert_eq!(ENV_AUDIT_LOGGER_QUEUE_CAPACITY, "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_values() {
|
||||
assert_eq!(DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY, 10000);
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_PRODUCTION, "production");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
|
||||
}
|
||||
}
|
||||
28
crates/config/src/observability/file.rs
Normal file
28
crates/config/src/observability/file.rs
Normal file
@@ -0,0 +1,28 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// RUSTFS_SINKS_FILE_PATH
|
||||
pub const ENV_SINKS_FILE_PATH: &str = "RUSTFS_SINKS_FILE_PATH";
|
||||
// RUSTFS_SINKS_FILE_BUFFER_SIZE
|
||||
pub const ENV_SINKS_FILE_BUFFER_SIZE: &str = "RUSTFS_SINKS_FILE_BUFFER_SIZE";
|
||||
// RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS
|
||||
pub const ENV_SINKS_FILE_FLUSH_INTERVAL_MS: &str = "RUSTFS_SINKS_FILE_FLUSH_INTERVAL_MS";
|
||||
// RUSTFS_SINKS_FILE_FLUSH_THRESHOLD
|
||||
pub const ENV_SINKS_FILE_FLUSH_THRESHOLD: &str = "RUSTFS_SINKS_FILE_FLUSH_THRESHOLD";
|
||||
|
||||
pub const DEFAULT_SINKS_FILE_BUFFER_SIZE: usize = 8192;
|
||||
|
||||
pub const DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS: u64 = 1000;
|
||||
|
||||
pub const DEFAULT_SINKS_FILE_FLUSH_THRESHOLD: usize = 100;
|
||||
27
crates/config/src/observability/kafka.rs
Normal file
27
crates/config/src/observability/kafka.rs
Normal file
@@ -0,0 +1,27 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// RUSTFS_SINKS_KAFKA_BROKERS
|
||||
pub const ENV_SINKS_KAFKA_BROKERS: &str = "RUSTFS_SINKS_KAFKA_BROKERS";
|
||||
pub const ENV_SINKS_KAFKA_TOPIC: &str = "RUSTFS_SINKS_KAFKA_TOPIC";
|
||||
// batch_size
|
||||
pub const ENV_SINKS_KAFKA_BATCH_SIZE: &str = "RUSTFS_SINKS_KAFKA_BATCH_SIZE";
|
||||
// batch_timeout_ms
|
||||
pub const ENV_SINKS_KAFKA_BATCH_TIMEOUT_MS: &str = "RUSTFS_SINKS_KAFKA_BATCH_TIMEOUT_MS";
|
||||
|
||||
// brokers
|
||||
pub const DEFAULT_SINKS_KAFKA_BROKERS: &str = "localhost:9092";
|
||||
pub const DEFAULT_SINKS_KAFKA_TOPIC: &str = "rustfs-sinks";
|
||||
pub const DEFAULT_SINKS_KAFKA_BATCH_SIZE: usize = 100;
|
||||
pub const DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS: u64 = 1000;
|
||||
@@ -12,87 +12,12 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// Observability Keys
|
||||
mod config;
|
||||
mod file;
|
||||
mod kafka;
|
||||
mod webhook;
|
||||
|
||||
pub const ENV_OBS_ENDPOINT: &str = "RUSTFS_OBS_ENDPOINT";
|
||||
pub const ENV_OBS_USE_STDOUT: &str = "RUSTFS_OBS_USE_STDOUT";
|
||||
pub const ENV_OBS_SAMPLE_RATIO: &str = "RUSTFS_OBS_SAMPLE_RATIO";
|
||||
pub const ENV_OBS_METER_INTERVAL: &str = "RUSTFS_OBS_METER_INTERVAL";
|
||||
pub const ENV_OBS_SERVICE_NAME: &str = "RUSTFS_OBS_SERVICE_NAME";
|
||||
pub const ENV_OBS_SERVICE_VERSION: &str = "RUSTFS_OBS_SERVICE_VERSION";
|
||||
pub const ENV_OBS_ENVIRONMENT: &str = "RUSTFS_OBS_ENVIRONMENT";
|
||||
pub const ENV_OBS_LOGGER_LEVEL: &str = "RUSTFS_OBS_LOGGER_LEVEL";
|
||||
pub const ENV_OBS_LOCAL_LOGGING_ENABLED: &str = "RUSTFS_OBS_LOCAL_LOGGING_ENABLED";
|
||||
pub const ENV_OBS_LOG_DIRECTORY: &str = "RUSTFS_OBS_LOG_DIRECTORY";
|
||||
pub const ENV_OBS_LOG_FILENAME: &str = "RUSTFS_OBS_LOG_FILENAME";
|
||||
pub const ENV_OBS_LOG_ROTATION_SIZE_MB: &str = "RUSTFS_OBS_LOG_ROTATION_SIZE_MB";
|
||||
pub const ENV_OBS_LOG_ROTATION_TIME: &str = "RUSTFS_OBS_LOG_ROTATION_TIME";
|
||||
pub const ENV_OBS_LOG_KEEP_FILES: &str = "RUSTFS_OBS_LOG_KEEP_FILES";
|
||||
|
||||
/// Log pool capacity for async logging
|
||||
pub const ENV_OBS_LOG_POOL_CAPA: &str = "RUSTFS_OBS_LOG_POOL_CAPA";
|
||||
|
||||
/// Log message capacity for async logging
|
||||
pub const ENV_OBS_LOG_MESSAGE_CAPA: &str = "RUSTFS_OBS_LOG_MESSAGE_CAPA";
|
||||
|
||||
/// Log flush interval in milliseconds for async logging
|
||||
pub const ENV_OBS_LOG_FLUSH_MS: &str = "RUSTFS_OBS_LOG_FLUSH_MS";
|
||||
|
||||
/// Default values for log pool
|
||||
pub const DEFAULT_OBS_LOG_POOL_CAPA: usize = 10240;
|
||||
|
||||
/// Default values for message capacity
|
||||
pub const DEFAULT_OBS_LOG_MESSAGE_CAPA: usize = 32768;
|
||||
|
||||
/// Default values for flush interval in milliseconds
|
||||
pub const DEFAULT_OBS_LOG_FLUSH_MS: u64 = 200;
|
||||
|
||||
/// Audit logger queue capacity environment variable key
|
||||
pub const ENV_AUDIT_LOGGER_QUEUE_CAPACITY: &str = "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY";
|
||||
|
||||
/// Default values for observability configuration
|
||||
pub const DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY: usize = 10000;
|
||||
|
||||
/// Default values for observability configuration
|
||||
// ### Supported Environment Values
|
||||
// - `production` - Secure file-only logging
|
||||
// - `development` - Full debugging with stdout
|
||||
// - `test` - Test environment with stdout support
|
||||
// - `staging` - Staging environment with stdout support
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_PRODUCTION: &str = "production";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT: &str = "development";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_TEST: &str = "test";
|
||||
pub const DEFAULT_OBS_ENVIRONMENT_STAGING: &str = "staging";
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_env_keys() {
|
||||
assert_eq!(ENV_OBS_ENDPOINT, "RUSTFS_OBS_ENDPOINT");
|
||||
assert_eq!(ENV_OBS_USE_STDOUT, "RUSTFS_OBS_USE_STDOUT");
|
||||
assert_eq!(ENV_OBS_SAMPLE_RATIO, "RUSTFS_OBS_SAMPLE_RATIO");
|
||||
assert_eq!(ENV_OBS_METER_INTERVAL, "RUSTFS_OBS_METER_INTERVAL");
|
||||
assert_eq!(ENV_OBS_SERVICE_NAME, "RUSTFS_OBS_SERVICE_NAME");
|
||||
assert_eq!(ENV_OBS_SERVICE_VERSION, "RUSTFS_OBS_SERVICE_VERSION");
|
||||
assert_eq!(ENV_OBS_ENVIRONMENT, "RUSTFS_OBS_ENVIRONMENT");
|
||||
assert_eq!(ENV_OBS_LOGGER_LEVEL, "RUSTFS_OBS_LOGGER_LEVEL");
|
||||
assert_eq!(ENV_OBS_LOCAL_LOGGING_ENABLED, "RUSTFS_OBS_LOCAL_LOGGING_ENABLED");
|
||||
assert_eq!(ENV_OBS_LOG_DIRECTORY, "RUSTFS_OBS_LOG_DIRECTORY");
|
||||
assert_eq!(ENV_OBS_LOG_FILENAME, "RUSTFS_OBS_LOG_FILENAME");
|
||||
assert_eq!(ENV_OBS_LOG_ROTATION_SIZE_MB, "RUSTFS_OBS_LOG_ROTATION_SIZE_MB");
|
||||
assert_eq!(ENV_OBS_LOG_ROTATION_TIME, "RUSTFS_OBS_LOG_ROTATION_TIME");
|
||||
assert_eq!(ENV_OBS_LOG_KEEP_FILES, "RUSTFS_OBS_LOG_KEEP_FILES");
|
||||
assert_eq!(ENV_AUDIT_LOGGER_QUEUE_CAPACITY, "RUSTFS_AUDIT_LOGGER_QUEUE_CAPACITY");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_default_values() {
|
||||
assert_eq!(DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY, 10000);
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_PRODUCTION, "production");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_DEVELOPMENT, "development");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_TEST, "test");
|
||||
assert_eq!(DEFAULT_OBS_ENVIRONMENT_STAGING, "staging");
|
||||
}
|
||||
}
|
||||
pub use config::*;
|
||||
pub use file::*;
|
||||
pub use kafka::*;
|
||||
pub use webhook::*;
|
||||
|
||||
28
crates/config/src/observability/webhook.rs
Normal file
28
crates/config/src/observability/webhook.rs
Normal file
@@ -0,0 +1,28 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
// RUSTFS_SINKS_WEBHOOK_ENDPOINT
|
||||
pub const ENV_SINKS_WEBHOOK_ENDPOINT: &str = "RUSTFS_SINKS_WEBHOOK_ENDPOINT";
|
||||
// RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN
|
||||
pub const ENV_SINKS_WEBHOOK_AUTH_TOKEN: &str = "RUSTFS_SINKS_WEBHOOK_AUTH_TOKEN";
|
||||
// max_retries
|
||||
pub const ENV_SINKS_WEBHOOK_MAX_RETRIES: &str = "RUSTFS_SINKS_WEBHOOK_MAX_RETRIES";
|
||||
// retry_delay_ms
|
||||
pub const ENV_SINKS_WEBHOOK_RETRY_DELAY_MS: &str = "RUSTFS_SINKS_WEBHOOK_RETRY_DELAY_MS";
|
||||
|
||||
// Default values for webhook sink configuration
|
||||
pub const DEFAULT_SINKS_WEBHOOK_ENDPOINT: &str = "http://localhost:8080";
|
||||
pub const DEFAULT_SINKS_WEBHOOK_AUTH_TOKEN: &str = "";
|
||||
pub const DEFAULT_SINKS_WEBHOOK_MAX_RETRIES: usize = 3;
|
||||
pub const DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS: u64 = 100;
|
||||
@@ -49,4 +49,4 @@ uuid = { workspace = true }
|
||||
base64 = { workspace = true }
|
||||
rand = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
md5 = { workspace = true }
|
||||
md5 = { workspace = true }
|
||||
|
||||
@@ -101,7 +101,6 @@ aws-credential-types = { workspace = true }
|
||||
aws-smithy-types = { workspace = true }
|
||||
parking_lot = { workspace = true }
|
||||
moka = { workspace = true }
|
||||
base64-simd.workspace = true
|
||||
|
||||
[target.'cfg(not(windows))'.dependencies]
|
||||
nix = { workspace = true }
|
||||
|
||||
@@ -17,10 +17,12 @@ pub mod datatypes;
|
||||
mod replication_pool;
|
||||
mod replication_resyncer;
|
||||
mod replication_state;
|
||||
mod replication_type;
|
||||
mod rule;
|
||||
|
||||
pub use config::*;
|
||||
pub use datatypes::*;
|
||||
pub use replication_pool::*;
|
||||
pub use replication_resyncer::*;
|
||||
pub use replication_type::*;
|
||||
pub use rule::*;
|
||||
|
||||
@@ -1,4 +1,9 @@
|
||||
use crate::StorageAPI;
|
||||
use crate::bucket::replication::MrfReplicateEntry;
|
||||
use crate::bucket::replication::ReplicateDecision;
|
||||
use crate::bucket::replication::ReplicateObjectInfo;
|
||||
use crate::bucket::replication::ReplicationWorkerOperation;
|
||||
use crate::bucket::replication::ResyncDecision;
|
||||
use crate::bucket::replication::ResyncOpts;
|
||||
use crate::bucket::replication::ResyncStatusType;
|
||||
use crate::bucket::replication::replicate_delete;
|
||||
@@ -13,21 +18,16 @@ use crate::bucket::replication::replication_resyncer::{
|
||||
BucketReplicationResyncStatus, DeletedObjectReplicationInfo, ReplicationResyncer,
|
||||
};
|
||||
use crate::bucket::replication::replication_state::ReplicationStats;
|
||||
use crate::bucket::replication::replication_statuses_map;
|
||||
use crate::bucket::replication::version_purge_statuses_map;
|
||||
use crate::config::com::read_config;
|
||||
use crate::error::Error as EcstoreError;
|
||||
use crate::store_api::ObjectInfo;
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_filemeta::MrfReplicateEntry;
|
||||
use rustfs_filemeta::ReplicateDecision;
|
||||
use rustfs_filemeta::ReplicateObjectInfo;
|
||||
use rustfs_filemeta::ReplicatedTargetInfo;
|
||||
use rustfs_filemeta::ReplicationStatusType;
|
||||
use rustfs_filemeta::ReplicationType;
|
||||
use rustfs_filemeta::ReplicationWorkerOperation;
|
||||
use rustfs_filemeta::ResyncDecision;
|
||||
use rustfs_filemeta::replication_statuses_map;
|
||||
use rustfs_filemeta::version_purge_statuses_map;
|
||||
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
@@ -996,7 +996,7 @@ pub async fn schedule_replication<S: StorageAPI>(oi: ObjectInfo, o: Arc<S>, dsc:
|
||||
target_purge_statuses: purge_statuses,
|
||||
replication_timestamp: tm,
|
||||
user_tags: oi.user_tags,
|
||||
checksum: None,
|
||||
checksum: vec![],
|
||||
retry_count: 0,
|
||||
event_type: "".to_string(),
|
||||
existing_obj_resync: ResyncDecision::default(),
|
||||
|
||||
@@ -2,8 +2,12 @@ use crate::bucket::bucket_target_sys::{
|
||||
AdvancedPutOptions, BucketTargetSys, PutObjectOptions, PutObjectPartOptions, RemoveObjectOptions, TargetClient,
|
||||
};
|
||||
use crate::bucket::metadata_sys;
|
||||
use crate::bucket::replication::ResyncStatusType;
|
||||
use crate::bucket::replication::{ObjectOpts, ReplicationConfigurationExt as _};
|
||||
use crate::bucket::replication::{MrfReplicateEntry, ReplicationWorkerOperation, ResyncStatusType};
|
||||
use crate::bucket::replication::{
|
||||
ObjectOpts, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATION_RESET, ReplicateObjectInfo,
|
||||
ReplicationConfigurationExt as _, ResyncTargetDecision, get_replication_state, parse_replicate_decision,
|
||||
replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
};
|
||||
use crate::bucket::tagging::decode_tags_to_map;
|
||||
use crate::bucket::target::BucketTargets;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
@@ -25,17 +29,14 @@ use byteorder::ByteOrder;
|
||||
use futures::future::join_all;
|
||||
use http::HeaderMap;
|
||||
|
||||
use regex::Regex;
|
||||
use rustfs_filemeta::{
|
||||
MrfReplicateEntry, REPLICATE_EXISTING, REPLICATE_EXISTING_DELETE, REPLICATION_RESET, ReplicateDecision, ReplicateObjectInfo,
|
||||
ReplicateTargetDecision, ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType,
|
||||
ReplicationType, ReplicationWorkerOperation, ResyncDecision, ResyncTargetDecision, VersionPurgeStatusType,
|
||||
get_replication_state, parse_replicate_decision, replication_statuses_map, target_reset_header, version_purge_statuses_map,
|
||||
ReplicatedInfos, ReplicatedTargetInfo, ReplicationAction, ReplicationState, ReplicationStatusType, ReplicationType,
|
||||
VersionPurgeStatusType,
|
||||
};
|
||||
use rustfs_utils::http::{
|
||||
AMZ_BUCKET_REPLICATION_STATUS, AMZ_OBJECT_TAGGING, AMZ_TAGGING_DIRECTIVE, CONTENT_ENCODING, HeaderExt as _,
|
||||
RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER, RUSTFS_REPLICATION_AUTUAL_OBJECT_SIZE,
|
||||
RUSTFS_REPLICATION_RESET_STATUS, SSEC_ALGORITHM_HEADER, SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, headers,
|
||||
RESERVED_METADATA_PREFIX, RESERVED_METADATA_PREFIX_LOWER, RUSTFS_REPLICATION_AUTUAL_OBJECT_SIZE, SSEC_ALGORITHM_HEADER,
|
||||
SSEC_KEY_HEADER, SSEC_KEY_MD5_HEADER, headers,
|
||||
};
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use rustfs_utils::string::strings_has_prefix_fold;
|
||||
@@ -55,6 +56,9 @@ use tokio::time::Duration as TokioDuration;
|
||||
use tokio_util::sync::CancellationToken;
|
||||
use tracing::{error, info, warn};
|
||||
|
||||
use super::replication_type::{ReplicateDecision, ReplicateTargetDecision, ResyncDecision};
|
||||
use regex::Regex;
|
||||
|
||||
const REPLICATION_DIR: &str = ".replication";
|
||||
const RESYNC_FILE_NAME: &str = "resync.bin";
|
||||
const RESYNC_META_FORMAT: u16 = 1;
|
||||
@@ -659,7 +663,7 @@ pub async fn get_heal_replicate_object_info(oi: &ObjectInfo, rcfg: &ReplicationC
|
||||
replication_timestamp: None,
|
||||
ssec: false, // TODO: add ssec support
|
||||
user_tags: oi.user_tags.clone(),
|
||||
checksum: oi.checksum.clone(),
|
||||
checksum: Vec::new(),
|
||||
retry_count: 0,
|
||||
}
|
||||
}
|
||||
@@ -845,7 +849,7 @@ impl ReplicationConfig {
|
||||
{
|
||||
resync_decision.targets.insert(
|
||||
decision.arn.clone(),
|
||||
resync_target(
|
||||
ResyncTargetDecision::resync_target(
|
||||
&oi,
|
||||
&target.arn,
|
||||
&target.reset_id,
|
||||
@@ -860,59 +864,6 @@ impl ReplicationConfig {
|
||||
}
|
||||
}
|
||||
|
||||
pub fn resync_target(
|
||||
oi: &ObjectInfo,
|
||||
arn: &str,
|
||||
reset_id: &str,
|
||||
reset_before_date: Option<OffsetDateTime>,
|
||||
status: ReplicationStatusType,
|
||||
) -> ResyncTargetDecision {
|
||||
let rs = oi
|
||||
.user_defined
|
||||
.get(target_reset_header(arn).as_str())
|
||||
.or(oi.user_defined.get(RUSTFS_REPLICATION_RESET_STATUS))
|
||||
.map(|s| s.to_string());
|
||||
|
||||
let mut dec = ResyncTargetDecision::default();
|
||||
|
||||
let mod_time = oi.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
|
||||
if rs.is_none() {
|
||||
let reset_before_date = reset_before_date.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
if !reset_id.is_empty() && mod_time < reset_before_date {
|
||||
dec.replicate = true;
|
||||
return dec;
|
||||
}
|
||||
|
||||
dec.replicate = status == ReplicationStatusType::Empty;
|
||||
|
||||
return dec;
|
||||
}
|
||||
|
||||
if reset_id.is_empty() || reset_before_date.is_none() {
|
||||
return dec;
|
||||
}
|
||||
|
||||
let rs = rs.unwrap();
|
||||
let reset_before_date = reset_before_date.unwrap();
|
||||
|
||||
let parts: Vec<&str> = rs.splitn(2, ';').collect();
|
||||
|
||||
if parts.len() != 2 {
|
||||
return dec;
|
||||
}
|
||||
|
||||
let new_reset = parts[0] == reset_id;
|
||||
|
||||
if !new_reset && status == ReplicationStatusType::Completed {
|
||||
return dec;
|
||||
}
|
||||
|
||||
dec.replicate = new_reset && mod_time < reset_before_date;
|
||||
|
||||
dec
|
||||
}
|
||||
|
||||
pub struct MustReplicateOptions {
|
||||
meta: HashMap<String, String>,
|
||||
status: ReplicationStatusType,
|
||||
@@ -982,7 +933,7 @@ pub async fn check_replicate_delete(
|
||||
let rcfg = match get_replication_config(bucket).await {
|
||||
Ok(Some(config)) => config,
|
||||
Ok(None) => {
|
||||
// warn!("No replication config found for bucket: {}", bucket);
|
||||
warn!("No replication config found for bucket: {}", bucket);
|
||||
return ReplicateDecision::default();
|
||||
}
|
||||
Err(err) => {
|
||||
|
||||
470
crates/ecstore/src/bucket/replication/replication_type.rs
Normal file
470
crates/ecstore/src/bucket/replication/replication_type.rs
Normal file
@@ -0,0 +1,470 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::error::{Error, Result};
|
||||
use crate::store_api::ObjectInfo;
|
||||
|
||||
use regex::Regex;
|
||||
|
||||
use rustfs_filemeta::VersionPurgeStatusType;
|
||||
use rustfs_filemeta::{ReplicatedInfos, ReplicationType};
|
||||
use rustfs_filemeta::{ReplicationState, ReplicationStatusType};
|
||||
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
|
||||
use rustfs_utils::http::RUSTFS_REPLICATION_RESET_STATUS;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::any::Any;
|
||||
use std::collections::HashMap;
|
||||
use std::fmt;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const REPLICATION_RESET: &str = "replication-reset";
|
||||
pub const REPLICATION_STATUS: &str = "replication-status";
|
||||
|
||||
// ReplicateQueued - replication being queued trail
|
||||
pub const REPLICATE_QUEUED: &str = "replicate:queue";
|
||||
|
||||
// ReplicateExisting - audit trail for existing objects replication
|
||||
pub const REPLICATE_EXISTING: &str = "replicate:existing";
|
||||
// ReplicateExistingDelete - audit trail for delete replication triggered for existing delete markers
|
||||
pub const REPLICATE_EXISTING_DELETE: &str = "replicate:existing:delete";
|
||||
|
||||
// ReplicateMRF - audit trail for replication from Most Recent Failures (MRF) queue
|
||||
pub const REPLICATE_MRF: &str = "replicate:mrf";
|
||||
// ReplicateIncoming - audit trail of inline replication
|
||||
pub const REPLICATE_INCOMING: &str = "replicate:incoming";
|
||||
// ReplicateIncomingDelete - audit trail of inline replication of deletes.
|
||||
pub const REPLICATE_INCOMING_DELETE: &str = "replicate:incoming:delete";
|
||||
|
||||
// ReplicateHeal - audit trail for healing of failed/pending replications
|
||||
pub const REPLICATE_HEAL: &str = "replicate:heal";
|
||||
// ReplicateHealDelete - audit trail of healing of failed/pending delete replications.
|
||||
pub const REPLICATE_HEAL_DELETE: &str = "replicate:heal:delete";
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct MrfReplicateEntry {
|
||||
#[serde(rename = "bucket")]
|
||||
pub bucket: String,
|
||||
|
||||
#[serde(rename = "object")]
|
||||
pub object: String,
|
||||
|
||||
#[serde(skip_serializing, skip_deserializing)]
|
||||
pub version_id: Option<Uuid>,
|
||||
|
||||
#[serde(rename = "retryCount")]
|
||||
pub retry_count: i32,
|
||||
|
||||
#[serde(skip_serializing, skip_deserializing)]
|
||||
pub size: i64,
|
||||
}
|
||||
|
||||
pub trait ReplicationWorkerOperation: Any + Send + Sync {
|
||||
fn to_mrf_entry(&self) -> MrfReplicateEntry;
|
||||
fn as_any(&self) -> &dyn Any;
|
||||
fn get_bucket(&self) -> &str;
|
||||
fn get_object(&self) -> &str;
|
||||
fn get_size(&self) -> i64;
|
||||
fn is_delete_marker(&self) -> bool;
|
||||
fn get_op_type(&self) -> ReplicationType;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ReplicateTargetDecision {
|
||||
pub replicate: bool,
|
||||
pub synchronous: bool,
|
||||
pub arn: String,
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
impl ReplicateTargetDecision {
|
||||
pub fn new(arn: String, replicate: bool, sync: bool) -> Self {
|
||||
Self {
|
||||
replicate,
|
||||
synchronous: sync,
|
||||
arn,
|
||||
id: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ReplicateTargetDecision {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{};{};{};{}", self.replicate, self.synchronous, self.arn, self.id)
|
||||
}
|
||||
}
|
||||
|
||||
/// ReplicateDecision represents replication decision for each target
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicateDecision {
|
||||
pub targets_map: HashMap<String, ReplicateTargetDecision>,
|
||||
}
|
||||
|
||||
impl ReplicateDecision {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
targets_map: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if at least one target qualifies for replication
|
||||
pub fn replicate_any(&self) -> bool {
|
||||
self.targets_map.values().any(|t| t.replicate)
|
||||
}
|
||||
|
||||
/// Returns true if at least one target qualifies for synchronous replication
|
||||
pub fn is_synchronous(&self) -> bool {
|
||||
self.targets_map.values().any(|t| t.synchronous)
|
||||
}
|
||||
|
||||
/// Updates ReplicateDecision with target's replication decision
|
||||
pub fn set(&mut self, target: ReplicateTargetDecision) {
|
||||
self.targets_map.insert(target.arn.clone(), target);
|
||||
}
|
||||
|
||||
/// Returns a stringified representation of internal replication status with all targets marked as `PENDING`
|
||||
pub fn pending_status(&self) -> Option<String> {
|
||||
let mut result = String::new();
|
||||
for target in self.targets_map.values() {
|
||||
if target.replicate {
|
||||
result.push_str(&format!("{}={};", target.arn, ReplicationStatusType::Pending.as_str()));
|
||||
}
|
||||
}
|
||||
if result.is_empty() { None } else { Some(result) }
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ReplicateDecision {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let mut result = String::new();
|
||||
for (key, value) in &self.targets_map {
|
||||
result.push_str(&format!("{key}={value},"));
|
||||
}
|
||||
write!(f, "{}", result.trim_end_matches(','))
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ReplicateDecision {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
// parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a
|
||||
// ReplicateDecision struct
|
||||
pub fn parse_replicate_decision(_bucket: &str, s: &str) -> Result<ReplicateDecision> {
|
||||
let mut decision = ReplicateDecision::new();
|
||||
|
||||
if s.is_empty() {
|
||||
return Ok(decision);
|
||||
}
|
||||
|
||||
for p in s.split(',') {
|
||||
if p.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let slc = p.split('=').collect::<Vec<&str>>();
|
||||
if slc.len() != 2 {
|
||||
return Err(Error::other(format!("invalid replicate decision format: {s}")));
|
||||
}
|
||||
|
||||
let tgt_str = slc[1].trim_matches('"');
|
||||
let tgt = tgt_str.split(';').collect::<Vec<&str>>();
|
||||
if tgt.len() != 4 {
|
||||
return Err(Error::other(format!("invalid replicate decision format: {s}")));
|
||||
}
|
||||
|
||||
let tgt = ReplicateTargetDecision {
|
||||
replicate: tgt[0] == "true",
|
||||
synchronous: tgt[1] == "true",
|
||||
arn: tgt[2].to_string(),
|
||||
id: tgt[3].to_string(),
|
||||
};
|
||||
decision.targets_map.insert(slc[0].to_string(), tgt);
|
||||
}
|
||||
|
||||
Ok(decision)
|
||||
|
||||
// r = ReplicateDecision{
|
||||
// targetsMap: make(map[string]replicateTargetDecision),
|
||||
// }
|
||||
// if len(s) == 0 {
|
||||
// return
|
||||
// }
|
||||
// for _, p := range strings.Split(s, ",") {
|
||||
// if p == "" {
|
||||
// continue
|
||||
// }
|
||||
// slc := strings.Split(p, "=")
|
||||
// if len(slc) != 2 {
|
||||
// return r, errInvalidReplicateDecisionFormat
|
||||
// }
|
||||
// tgtStr := strings.TrimSuffix(strings.TrimPrefix(slc[1], `"`), `"`)
|
||||
// tgt := strings.Split(tgtStr, ";")
|
||||
// if len(tgt) != 4 {
|
||||
// return r, errInvalidReplicateDecisionFormat
|
||||
// }
|
||||
// r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: tgt[0] == "true", Synchronous: tgt[1] == "true", Arn: tgt[2], ID: tgt[3]}
|
||||
// }
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ResyncTargetDecision {
|
||||
pub replicate: bool,
|
||||
pub reset_id: String,
|
||||
pub reset_before_date: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
pub fn target_reset_header(arn: &str) -> String {
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-{arn}")
|
||||
}
|
||||
|
||||
impl ResyncTargetDecision {
|
||||
pub fn resync_target(
|
||||
oi: &ObjectInfo,
|
||||
arn: &str,
|
||||
reset_id: &str,
|
||||
reset_before_date: Option<OffsetDateTime>,
|
||||
status: ReplicationStatusType,
|
||||
) -> Self {
|
||||
let rs = oi
|
||||
.user_defined
|
||||
.get(target_reset_header(arn).as_str())
|
||||
.or(oi.user_defined.get(RUSTFS_REPLICATION_RESET_STATUS))
|
||||
.map(|s| s.to_string());
|
||||
|
||||
let mut dec = Self::default();
|
||||
|
||||
let mod_time = oi.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
|
||||
if rs.is_none() {
|
||||
let reset_before_date = reset_before_date.unwrap_or(OffsetDateTime::UNIX_EPOCH);
|
||||
if !reset_id.is_empty() && mod_time < reset_before_date {
|
||||
dec.replicate = true;
|
||||
return dec;
|
||||
}
|
||||
|
||||
dec.replicate = status == ReplicationStatusType::Empty;
|
||||
|
||||
return dec;
|
||||
}
|
||||
|
||||
if reset_id.is_empty() || reset_before_date.is_none() {
|
||||
return dec;
|
||||
}
|
||||
|
||||
let rs = rs.unwrap();
|
||||
let reset_before_date = reset_before_date.unwrap();
|
||||
|
||||
let parts: Vec<&str> = rs.splitn(2, ';').collect();
|
||||
|
||||
if parts.len() != 2 {
|
||||
return dec;
|
||||
}
|
||||
|
||||
let new_reset = parts[0] == reset_id;
|
||||
|
||||
if !new_reset && status == ReplicationStatusType::Completed {
|
||||
return dec;
|
||||
}
|
||||
|
||||
dec.replicate = new_reset && mod_time < reset_before_date;
|
||||
|
||||
dec
|
||||
}
|
||||
}
|
||||
|
||||
/// ResyncDecision is a struct representing a map with target's individual resync decisions
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResyncDecision {
|
||||
pub targets: HashMap<String, ResyncTargetDecision>,
|
||||
}
|
||||
|
||||
impl ResyncDecision {
|
||||
pub fn new() -> Self {
|
||||
Self { targets: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Returns true if no targets with resync decision present
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.targets.is_empty()
|
||||
}
|
||||
|
||||
pub fn must_resync(&self) -> bool {
|
||||
self.targets.values().any(|v| v.replicate)
|
||||
}
|
||||
|
||||
pub fn must_resync_target(&self, tgt_arn: &str) -> bool {
|
||||
self.targets.get(tgt_arn).map(|v| v.replicate).unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ResyncDecision {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicateObjectInfo {
|
||||
pub name: String,
|
||||
pub size: i64,
|
||||
pub actual_size: i64,
|
||||
pub bucket: String,
|
||||
pub version_id: Option<Uuid>,
|
||||
pub etag: Option<String>,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
pub replication_status: ReplicationStatusType,
|
||||
pub replication_status_internal: Option<String>,
|
||||
pub delete_marker: bool,
|
||||
pub version_purge_status_internal: Option<String>,
|
||||
pub version_purge_status: VersionPurgeStatusType,
|
||||
pub replication_state: Option<ReplicationState>,
|
||||
pub op_type: ReplicationType,
|
||||
pub event_type: String,
|
||||
pub dsc: ReplicateDecision,
|
||||
pub existing_obj_resync: ResyncDecision,
|
||||
pub target_statuses: HashMap<String, ReplicationStatusType>,
|
||||
pub target_purge_statuses: HashMap<String, VersionPurgeStatusType>,
|
||||
pub replication_timestamp: Option<OffsetDateTime>,
|
||||
pub ssec: bool,
|
||||
pub user_tags: String,
|
||||
pub checksum: Vec<u8>,
|
||||
pub retry_count: u32,
|
||||
}
|
||||
|
||||
impl ReplicationWorkerOperation for ReplicateObjectInfo {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn to_mrf_entry(&self) -> MrfReplicateEntry {
|
||||
MrfReplicateEntry {
|
||||
bucket: self.bucket.clone(),
|
||||
object: self.name.clone(),
|
||||
version_id: self.version_id,
|
||||
retry_count: self.retry_count as i32,
|
||||
size: self.size,
|
||||
}
|
||||
}
|
||||
|
||||
fn get_bucket(&self) -> &str {
|
||||
&self.bucket
|
||||
}
|
||||
|
||||
fn get_object(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn get_size(&self) -> i64 {
|
||||
self.size
|
||||
}
|
||||
|
||||
fn is_delete_marker(&self) -> bool {
|
||||
self.delete_marker
|
||||
}
|
||||
|
||||
fn get_op_type(&self) -> ReplicationType {
|
||||
self.op_type
|
||||
}
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref REPL_STATUS_REGEX: Regex = Regex::new(r"([^=].*?)=([^,].*?);").unwrap();
|
||||
}
|
||||
|
||||
impl ReplicateObjectInfo {
|
||||
/// Returns replication status of a target
|
||||
pub fn target_replication_status(&self, arn: &str) -> ReplicationStatusType {
|
||||
let binding = self.replication_status_internal.clone().unwrap_or_default();
|
||||
let captures = REPL_STATUS_REGEX.captures_iter(&binding);
|
||||
for cap in captures {
|
||||
if cap.len() == 3 && &cap[1] == arn {
|
||||
return ReplicationStatusType::from(&cap[2]);
|
||||
}
|
||||
}
|
||||
ReplicationStatusType::default()
|
||||
}
|
||||
|
||||
/// Returns the relevant info needed by MRF
|
||||
pub fn to_mrf_entry(&self) -> MrfReplicateEntry {
|
||||
MrfReplicateEntry {
|
||||
bucket: self.bucket.clone(),
|
||||
object: self.name.clone(),
|
||||
version_id: self.version_id,
|
||||
retry_count: self.retry_count as i32,
|
||||
size: self.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// constructs a replication status map from string representation
|
||||
pub fn replication_statuses_map(s: &str) -> HashMap<String, ReplicationStatusType> {
|
||||
let mut targets = HashMap::new();
|
||||
let rep_stat_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
|
||||
for (_, [arn, status]) in rep_stat_matches {
|
||||
if arn.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let status = ReplicationStatusType::from(status);
|
||||
targets.insert(arn.to_string(), status);
|
||||
}
|
||||
targets
|
||||
}
|
||||
|
||||
// constructs a version purge status map from string representation
|
||||
pub fn version_purge_statuses_map(s: &str) -> HashMap<String, VersionPurgeStatusType> {
|
||||
let mut targets = HashMap::new();
|
||||
let purge_status_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
|
||||
for (_, [arn, status]) in purge_status_matches {
|
||||
if arn.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let status = VersionPurgeStatusType::from(status);
|
||||
targets.insert(arn.to_string(), status);
|
||||
}
|
||||
targets
|
||||
}
|
||||
|
||||
pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationState, _vid: Option<String>) -> ReplicationState {
|
||||
let reset_status_map: Vec<(String, String)> = rinfos
|
||||
.targets
|
||||
.iter()
|
||||
.filter(|v| !v.resync_timestamp.is_empty())
|
||||
.map(|t| (target_reset_header(t.arn.as_str()), t.resync_timestamp.clone()))
|
||||
.collect();
|
||||
|
||||
let repl_statuses = rinfos.replication_status_internal();
|
||||
let vpurge_statuses = rinfos.version_purge_status_internal();
|
||||
|
||||
let mut reset_statuses_map = prev_state.reset_statuses_map.clone();
|
||||
for (key, value) in reset_status_map {
|
||||
reset_statuses_map.insert(key, value);
|
||||
}
|
||||
|
||||
ReplicationState {
|
||||
replicate_decision_str: prev_state.replicate_decision_str.clone(),
|
||||
reset_statuses_map,
|
||||
replica_timestamp: prev_state.replica_timestamp,
|
||||
replica_status: prev_state.replica_status.clone(),
|
||||
targets: replication_statuses_map(&repl_statuses.clone().unwrap_or_default()),
|
||||
replication_status_internal: repl_statuses,
|
||||
replication_timestamp: rinfos.replication_timestamp,
|
||||
purge_targets: version_purge_statuses_map(&vpurge_statuses.clone().unwrap_or_default()),
|
||||
version_purge_status_internal: vpurge_statuses,
|
||||
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -30,8 +30,7 @@ use s3s::header::{
|
||||
X_AMZ_STORAGE_CLASS, X_AMZ_WEBSITE_REDIRECT_LOCATION,
|
||||
};
|
||||
//use crate::disk::{BufferReader, Reader};
|
||||
use crate::client::checksum::ChecksumMode;
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::checksum::ChecksumMode;
|
||||
use crate::client::{
|
||||
api_error_response::{err_entity_too_large, err_invalid_argument},
|
||||
api_put_object_common::optimal_part_info,
|
||||
@@ -42,6 +41,7 @@ use crate::client::{
|
||||
transition_api::{ReaderImpl, TransitionClient, UploadInfo},
|
||||
utils::{is_amz_header, is_minio_header, is_rustfs_header, is_standard_header, is_storageclass_header},
|
||||
};
|
||||
use rustfs_utils::crypto::base64_encode;
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AdvancedPutOptions {
|
||||
|
||||
@@ -25,8 +25,7 @@ use time::OffsetDateTime;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::client::checksum::ChecksumMode;
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::checksum::ChecksumMode;
|
||||
use crate::client::{
|
||||
api_error_response::{
|
||||
err_entity_too_large, err_entity_too_small, err_invalid_argument, http_resp_to_error_response, to_error_response,
|
||||
@@ -39,7 +38,7 @@ use crate::client::{
|
||||
constants::{ISO8601_DATEFORMAT, MAX_PART_SIZE, MAX_SINGLE_PUT_OBJECT_SIZE},
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
|
||||
};
|
||||
use rustfs_utils::path::trim_etag;
|
||||
use rustfs_utils::{crypto::base64_encode, path::trim_etag};
|
||||
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
|
||||
|
||||
impl TransitionClient {
|
||||
|
||||
@@ -29,7 +29,7 @@ use tokio_util::sync::CancellationToken;
|
||||
use tracing::warn;
|
||||
use uuid::Uuid;
|
||||
|
||||
use crate::client::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
|
||||
use crate::checksum::{ChecksumMode, add_auto_checksum_headers, apply_auto_checksum};
|
||||
use crate::client::{
|
||||
api_error_response::{err_invalid_argument, err_unexpected_eof, http_resp_to_error_response},
|
||||
api_put_object::PutObjectOptions,
|
||||
@@ -40,8 +40,7 @@ use crate::client::{
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient, UploadInfo},
|
||||
};
|
||||
|
||||
use crate::client::utils::base64_encode;
|
||||
use rustfs_utils::path::trim_etag;
|
||||
use rustfs_utils::{crypto::base64_encode, path::trim_etag};
|
||||
use s3s::header::{X_AMZ_EXPIRATION, X_AMZ_VERSION_ID};
|
||||
|
||||
pub struct UploadedPartRes {
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue, Method, StatusCode};
|
||||
use rustfs_utils::HashAlgorithm;
|
||||
use rustfs_utils::{HashAlgorithm, crypto::base64_encode};
|
||||
use s3s::S3ErrorCode;
|
||||
use s3s::dto::ReplicationStatus;
|
||||
use s3s::header::X_AMZ_BYPASS_GOVERNANCE_RETENTION;
|
||||
@@ -29,7 +29,6 @@ use std::{collections::HashMap, sync::Arc};
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::client::{
|
||||
api_error_response::{ErrorResponse, http_resp_to_error_response, to_error_response},
|
||||
transition_api::{ReaderImpl, RequestMetadata, TransitionClient},
|
||||
|
||||
@@ -23,9 +23,9 @@ use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use time::OffsetDateTime;
|
||||
|
||||
use crate::client::checksum::ChecksumMode;
|
||||
use crate::checksum::ChecksumMode;
|
||||
use crate::client::transition_api::ObjectMultipartInfo;
|
||||
use crate::client::utils::base64_decode;
|
||||
use rustfs_utils::crypto::base64_decode;
|
||||
|
||||
use super::transition_api;
|
||||
|
||||
|
||||
@@ -1,351 +0,0 @@
|
||||
#![allow(clippy::map_entry)]
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
#![allow(unused_imports)]
|
||||
#![allow(unused_variables)]
|
||||
#![allow(unused_mut)]
|
||||
#![allow(unused_assignments)]
|
||||
#![allow(unused_must_use)]
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use lazy_static::lazy_static;
|
||||
use rustfs_checksums::ChecksumAlgorithm;
|
||||
use std::collections::HashMap;
|
||||
|
||||
use crate::client::utils::base64_decode;
|
||||
use crate::client::utils::base64_encode;
|
||||
use crate::client::{api_put_object::PutObjectOptions, api_s3_datatypes::ObjectPart};
|
||||
use crate::{disk::DiskAPI, store_api::GetObjectReader};
|
||||
use s3s::header::{
|
||||
X_AMZ_CHECKSUM_ALGORITHM, X_AMZ_CHECKSUM_CRC32, X_AMZ_CHECKSUM_CRC32C, X_AMZ_CHECKSUM_SHA1, X_AMZ_CHECKSUM_SHA256,
|
||||
};
|
||||
|
||||
use enumset::{EnumSet, EnumSetType, enum_set};
|
||||
|
||||
#[derive(Debug, EnumSetType, Default)]
|
||||
#[enumset(repr = "u8")]
|
||||
pub enum ChecksumMode {
|
||||
#[default]
|
||||
ChecksumNone,
|
||||
ChecksumSHA256,
|
||||
ChecksumSHA1,
|
||||
ChecksumCRC32,
|
||||
ChecksumCRC32C,
|
||||
ChecksumCRC64NVME,
|
||||
ChecksumFullObject,
|
||||
}
|
||||
|
||||
lazy_static! {
|
||||
static ref C_ChecksumMask: EnumSet<ChecksumMode> = {
|
||||
let mut s = EnumSet::all();
|
||||
s.remove(ChecksumMode::ChecksumFullObject);
|
||||
s
|
||||
};
|
||||
static ref C_ChecksumFullObjectCRC32: EnumSet<ChecksumMode> =
|
||||
enum_set!(ChecksumMode::ChecksumCRC32 | ChecksumMode::ChecksumFullObject);
|
||||
static ref C_ChecksumFullObjectCRC32C: EnumSet<ChecksumMode> =
|
||||
enum_set!(ChecksumMode::ChecksumCRC32C | ChecksumMode::ChecksumFullObject);
|
||||
}
|
||||
const AMZ_CHECKSUM_CRC64NVME: &str = "x-amz-checksum-crc64nvme";
|
||||
|
||||
impl ChecksumMode {
|
||||
//pub const CRC64_NVME_POLYNOMIAL: i64 = 0xad93d23594c93659;
|
||||
|
||||
pub fn base(&self) -> ChecksumMode {
|
||||
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
|
||||
match s.as_u8() {
|
||||
1_u8 => ChecksumMode::ChecksumNone,
|
||||
2_u8 => ChecksumMode::ChecksumSHA256,
|
||||
4_u8 => ChecksumMode::ChecksumSHA1,
|
||||
8_u8 => ChecksumMode::ChecksumCRC32,
|
||||
16_u8 => ChecksumMode::ChecksumCRC32C,
|
||||
32_u8 => ChecksumMode::ChecksumCRC64NVME,
|
||||
_ => panic!("enum err."),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is(&self, t: ChecksumMode) -> bool {
|
||||
*self & t == t
|
||||
}
|
||||
|
||||
pub fn key(&self) -> String {
|
||||
//match c & checksumMask {
|
||||
match self {
|
||||
ChecksumMode::ChecksumCRC32 => {
|
||||
return X_AMZ_CHECKSUM_CRC32.to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
return X_AMZ_CHECKSUM_CRC32C.to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA1 => {
|
||||
return X_AMZ_CHECKSUM_SHA1.to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA256 => {
|
||||
return X_AMZ_CHECKSUM_SHA256.to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC64NVME => {
|
||||
return AMZ_CHECKSUM_CRC64NVME.to_string();
|
||||
}
|
||||
_ => {
|
||||
return "".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub fn can_composite(&self) -> bool {
|
||||
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
|
||||
match s.as_u8() {
|
||||
2_u8 => true,
|
||||
4_u8 => true,
|
||||
8_u8 => true,
|
||||
16_u8 => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn can_merge_crc(&self) -> bool {
|
||||
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
|
||||
match s.as_u8() {
|
||||
8_u8 => true,
|
||||
16_u8 => true,
|
||||
32_u8 => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn full_object_requested(&self) -> bool {
|
||||
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
|
||||
match s.as_u8() {
|
||||
//C_ChecksumFullObjectCRC32 as u8 => true,
|
||||
//C_ChecksumFullObjectCRC32C as u8 => true,
|
||||
32_u8 => true,
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn key_capitalized(&self) -> String {
|
||||
self.key()
|
||||
}
|
||||
|
||||
pub fn raw_byte_len(&self) -> usize {
|
||||
let u = EnumSet::from(*self).intersection(*C_ChecksumMask).as_u8();
|
||||
if u == ChecksumMode::ChecksumCRC32 as u8 || u == ChecksumMode::ChecksumCRC32C as u8 {
|
||||
4
|
||||
} else if u == ChecksumMode::ChecksumSHA1 as u8 {
|
||||
use sha1::Digest;
|
||||
sha1::Sha1::output_size() as usize
|
||||
} else if u == ChecksumMode::ChecksumSHA256 as u8 {
|
||||
use sha2::Digest;
|
||||
sha2::Sha256::output_size() as usize
|
||||
} else if u == ChecksumMode::ChecksumCRC64NVME as u8 {
|
||||
8
|
||||
} else {
|
||||
0
|
||||
}
|
||||
}
|
||||
|
||||
pub fn hasher(&self) -> Result<Box<dyn rustfs_checksums::http::HttpChecksum>, std::io::Error> {
|
||||
match /*C_ChecksumMask & **/self {
|
||||
ChecksumMode::ChecksumCRC32 => {
|
||||
return Ok(ChecksumAlgorithm::Crc32.into_impl());
|
||||
}
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
return Ok(ChecksumAlgorithm::Crc32c.into_impl());
|
||||
}
|
||||
ChecksumMode::ChecksumSHA1 => {
|
||||
return Ok(ChecksumAlgorithm::Sha1.into_impl());
|
||||
}
|
||||
ChecksumMode::ChecksumSHA256 => {
|
||||
return Ok(ChecksumAlgorithm::Sha256.into_impl());
|
||||
}
|
||||
ChecksumMode::ChecksumCRC64NVME => {
|
||||
return Ok(ChecksumAlgorithm::Crc64Nvme.into_impl());
|
||||
}
|
||||
_ => return Err(std::io::Error::other("unsupported checksum type")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn is_set(&self) -> bool {
|
||||
let s = EnumSet::from(*self).intersection(*C_ChecksumMask);
|
||||
s.len() == 1
|
||||
}
|
||||
|
||||
pub fn set_default(&mut self, t: ChecksumMode) {
|
||||
if !self.is_set() {
|
||||
*self = t;
|
||||
}
|
||||
}
|
||||
|
||||
pub fn encode_to_string(&self, b: &[u8]) -> Result<String, std::io::Error> {
|
||||
if !self.is_set() {
|
||||
return Ok("".to_string());
|
||||
}
|
||||
let mut h = self.hasher()?;
|
||||
h.update(b);
|
||||
let hash = h.finalize();
|
||||
Ok(base64_encode(hash.as_ref()))
|
||||
}
|
||||
|
||||
pub fn to_string(&self) -> String {
|
||||
//match c & checksumMask {
|
||||
match self {
|
||||
ChecksumMode::ChecksumCRC32 => {
|
||||
return "CRC32".to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC32C => {
|
||||
return "CRC32C".to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA1 => {
|
||||
return "SHA1".to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumSHA256 => {
|
||||
return "SHA256".to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumNone => {
|
||||
return "".to_string();
|
||||
}
|
||||
ChecksumMode::ChecksumCRC64NVME => {
|
||||
return "CRC64NVME".to_string();
|
||||
}
|
||||
_ => {
|
||||
return "<invalid>".to_string();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// pub fn check_sum_reader(&self, r: GetObjectReader) -> Result<Checksum, std::io::Error> {
|
||||
// let mut h = self.hasher()?;
|
||||
// Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
|
||||
// }
|
||||
|
||||
// pub fn check_sum_bytes(&self, b: &[u8]) -> Result<Checksum, std::io::Error> {
|
||||
// let mut h = self.hasher()?;
|
||||
// Ok(Checksum::new(self.clone(), h.sum().as_bytes()))
|
||||
// }
|
||||
|
||||
pub fn composite_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
|
||||
if !self.can_composite() {
|
||||
return Err(std::io::Error::other("cannot do composite checksum"));
|
||||
}
|
||||
p.sort_by(|i, j| {
|
||||
if i.part_num < j.part_num {
|
||||
std::cmp::Ordering::Less
|
||||
} else if i.part_num > j.part_num {
|
||||
std::cmp::Ordering::Greater
|
||||
} else {
|
||||
std::cmp::Ordering::Equal
|
||||
}
|
||||
});
|
||||
let c = self.base();
|
||||
let crc_bytes = Vec::<u8>::with_capacity(p.len() * self.raw_byte_len() as usize);
|
||||
let mut h = self.hasher()?;
|
||||
h.update(crc_bytes.as_ref());
|
||||
let hash = h.finalize();
|
||||
Ok(Checksum {
|
||||
checksum_type: self.clone(),
|
||||
r: hash.as_ref().to_vec(),
|
||||
computed: false,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn full_object_checksum(&self, p: &mut [ObjectPart]) -> Result<Checksum, std::io::Error> {
|
||||
todo!();
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct Checksum {
|
||||
checksum_type: ChecksumMode,
|
||||
r: Vec<u8>,
|
||||
computed: bool,
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
impl Checksum {
|
||||
fn new(t: ChecksumMode, b: &[u8]) -> Checksum {
|
||||
if t.is_set() && b.len() == t.raw_byte_len() {
|
||||
return Checksum {
|
||||
checksum_type: t,
|
||||
r: b.to_vec(),
|
||||
computed: false,
|
||||
};
|
||||
}
|
||||
Checksum::default()
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn new_checksum_string(t: ChecksumMode, s: &str) -> Result<Checksum, std::io::Error> {
|
||||
let b = match base64_decode(s.as_bytes()) {
|
||||
Ok(b) => b,
|
||||
Err(err) => return Err(std::io::Error::other(err.to_string())),
|
||||
};
|
||||
if t.is_set() && b.len() == t.raw_byte_len() {
|
||||
return Ok(Checksum {
|
||||
checksum_type: t,
|
||||
r: b,
|
||||
computed: false,
|
||||
});
|
||||
}
|
||||
Ok(Checksum::default())
|
||||
}
|
||||
|
||||
fn is_set(&self) -> bool {
|
||||
self.checksum_type.is_set() && self.r.len() == self.checksum_type.raw_byte_len()
|
||||
}
|
||||
|
||||
fn encoded(&self) -> String {
|
||||
if !self.is_set() {
|
||||
return "".to_string();
|
||||
}
|
||||
base64_encode(&self.r)
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
fn raw(&self) -> Option<Vec<u8>> {
|
||||
if !self.is_set() {
|
||||
return None;
|
||||
}
|
||||
Some(self.r.clone())
|
||||
}
|
||||
}
|
||||
|
||||
pub fn add_auto_checksum_headers(opts: &mut PutObjectOptions) {
|
||||
opts.user_metadata
|
||||
.insert("X-Amz-Checksum-Algorithm".to_string(), opts.auto_checksum.to_string());
|
||||
if opts.auto_checksum.full_object_requested() {
|
||||
opts.user_metadata
|
||||
.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
|
||||
}
|
||||
}
|
||||
|
||||
pub fn apply_auto_checksum(opts: &mut PutObjectOptions, all_parts: &mut [ObjectPart]) -> Result<(), std::io::Error> {
|
||||
if opts.auto_checksum.can_composite() && !opts.auto_checksum.is(ChecksumMode::ChecksumFullObject) {
|
||||
let crc = opts.auto_checksum.composite_checksum(all_parts)?;
|
||||
opts.user_metadata = {
|
||||
let mut hm = HashMap::new();
|
||||
hm.insert(opts.auto_checksum.key(), crc.encoded());
|
||||
hm
|
||||
}
|
||||
} else if opts.auto_checksum.can_merge_crc() {
|
||||
let crc = opts.auto_checksum.full_object_checksum(all_parts)?;
|
||||
opts.user_metadata = {
|
||||
let mut hm = HashMap::new();
|
||||
hm.insert(opts.auto_checksum.key_capitalized(), crc.encoded());
|
||||
hm.insert("X-Amz-Checksum-Type".to_string(), "FULL_OBJECT".to_string());
|
||||
hm
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -30,7 +30,6 @@ pub mod api_restore;
|
||||
pub mod api_s3_datatypes;
|
||||
pub mod api_stat;
|
||||
pub mod bucket_cache;
|
||||
pub mod checksum;
|
||||
pub mod constants;
|
||||
pub mod credentials;
|
||||
pub mod object_api_utils;
|
||||
|
||||
@@ -20,8 +20,8 @@
|
||||
#![allow(clippy::all)]
|
||||
|
||||
use http::HeaderMap;
|
||||
use s3s::dto::ETag;
|
||||
use std::{collections::HashMap, io::Cursor, sync::Arc};
|
||||
use std::io::Cursor;
|
||||
use std::{collections::HashMap, sync::Arc};
|
||||
use tokio::io::BufReader;
|
||||
|
||||
use crate::error::ErrorResponse;
|
||||
@@ -148,30 +148,27 @@ pub fn new_getobjectreader(
|
||||
Ok((get_fn, off as i64, length as i64))
|
||||
}
|
||||
|
||||
/// Convert a raw stored ETag into the strongly-typed `s3s::dto::ETag`.
|
||||
///
|
||||
/// Supports already quoted (`"abc"`), weak (`W/"abc"`), or plain (`abc`) values.
|
||||
pub fn to_s3s_etag(etag: &str) -> ETag {
|
||||
if let Some(rest) = etag.strip_prefix("W/\"") {
|
||||
if let Some(body) = rest.strip_suffix('"') {
|
||||
return ETag::Weak(body.to_string());
|
||||
}
|
||||
return ETag::Weak(rest.to_string());
|
||||
/// Format an ETag value according to HTTP standards (wrap with quotes if not already wrapped)
|
||||
pub fn format_etag(etag: &str) -> String {
|
||||
if etag.starts_with('"') && etag.ends_with('"') {
|
||||
// Already properly formatted
|
||||
etag.to_string()
|
||||
} else if etag.starts_with("W/\"") && etag.ends_with('"') {
|
||||
// Already a weak ETag, properly formatted
|
||||
etag.to_string()
|
||||
} else {
|
||||
// Need to wrap with quotes
|
||||
format!("\"{}\"", etag)
|
||||
}
|
||||
|
||||
if let Some(body) = etag.strip_prefix('"').and_then(|rest| rest.strip_suffix('"')) {
|
||||
return ETag::Strong(body.to_string());
|
||||
}
|
||||
|
||||
ETag::Strong(etag.to_string())
|
||||
}
|
||||
|
||||
pub fn get_raw_etag(metadata: &HashMap<String, String>) -> String {
|
||||
metadata
|
||||
.get("etag")
|
||||
.cloned()
|
||||
.or_else(|| metadata.get("md5Sum").cloned())
|
||||
.unwrap_or_default()
|
||||
pub fn extract_etag(metadata: &HashMap<String, String>) -> String {
|
||||
let etag = if let Some(etag) = metadata.get("etag") {
|
||||
etag.clone()
|
||||
} else {
|
||||
metadata["md5Sum"].clone()
|
||||
};
|
||||
format_etag(&etag)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -179,28 +176,30 @@ mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_to_s3s_etag() {
|
||||
// Test unquoted ETag - should become strong etag
|
||||
fn test_format_etag() {
|
||||
// Test unquoted ETag - should add quotes
|
||||
assert_eq!(format_etag("6af8d12c0c74b78094884349f3c8a079"), "\"6af8d12c0c74b78094884349f3c8a079\"");
|
||||
|
||||
// Test already quoted ETag - should not double quote
|
||||
assert_eq!(
|
||||
to_s3s_etag("6af8d12c0c74b78094884349f3c8a079"),
|
||||
ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
format_etag("\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
"\"6af8d12c0c74b78094884349f3c8a079\""
|
||||
);
|
||||
|
||||
// Test weak ETag - should keep as is
|
||||
assert_eq!(
|
||||
to_s3s_etag("\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
ETag::Strong("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
format_etag("W/\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
"W/\"6af8d12c0c74b78094884349f3c8a079\""
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
to_s3s_etag("W/\"6af8d12c0c74b78094884349f3c8a079\""),
|
||||
ETag::Weak("6af8d12c0c74b78094884349f3c8a079".to_string())
|
||||
);
|
||||
// Test empty ETag - should add quotes
|
||||
assert_eq!(format_etag(""), "\"\"");
|
||||
|
||||
assert_eq!(to_s3s_etag(""), ETag::Strong(String::new()));
|
||||
// Test malformed quote (only starting quote) - should wrap properly
|
||||
assert_eq!(format_etag("\"incomplete"), "\"\"incomplete\"");
|
||||
|
||||
assert_eq!(to_s3s_etag("\"incomplete"), ETag::Strong("\"incomplete".to_string()));
|
||||
|
||||
assert_eq!(to_s3s_etag("incomplete\""), ETag::Strong("incomplete\"".to_string()));
|
||||
// Test malformed quote (only ending quote) - should wrap properly
|
||||
assert_eq!(format_etag("incomplete\""), "\"incomplete\"\"");
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -209,17 +208,15 @@ mod tests {
|
||||
|
||||
// Test with etag field
|
||||
metadata.insert("etag".to_string(), "abc123".to_string());
|
||||
assert_eq!(get_raw_etag(&metadata), "abc123");
|
||||
assert_eq!(extract_etag(&metadata), "\"abc123\"");
|
||||
|
||||
// Test with already quoted etag field
|
||||
metadata.insert("etag".to_string(), "\"def456\"".to_string());
|
||||
assert_eq!(get_raw_etag(&metadata), "\"def456\"");
|
||||
assert_eq!(extract_etag(&metadata), "\"def456\"");
|
||||
|
||||
// Test fallback to md5Sum
|
||||
metadata.remove("etag");
|
||||
metadata.insert("md5Sum".to_string(), "xyz789".to_string());
|
||||
assert_eq!(get_raw_etag(&metadata), "xyz789");
|
||||
|
||||
metadata.clear();
|
||||
assert_eq!(get_raw_etag(&metadata), "");
|
||||
assert_eq!(extract_etag(&metadata), "\"xyz789\"");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -61,7 +61,7 @@ use crate::client::{
|
||||
constants::{UNSIGNED_PAYLOAD, UNSIGNED_PAYLOAD_TRAILER},
|
||||
credentials::{CredContext, Credentials, SignatureType, Static},
|
||||
};
|
||||
use crate::{client::checksum::ChecksumMode, store_api::GetObjectReader};
|
||||
use crate::{checksum::ChecksumMode, store_api::GetObjectReader};
|
||||
use rustfs_rio::HashReader;
|
||||
use rustfs_utils::{
|
||||
net::get_endpoint_url,
|
||||
|
||||
@@ -90,11 +90,3 @@ pub fn is_rustfs_header(header_key: &str) -> bool {
|
||||
pub fn is_minio_header(header_key: &str) -> bool {
|
||||
header_key.to_lowercase().starts_with("x-minio-")
|
||||
}
|
||||
|
||||
pub fn base64_encode(input: &[u8]) -> String {
|
||||
base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
|
||||
}
|
||||
|
||||
pub fn base64_decode(input: &[u8]) -> Result<Vec<u8>, base64_simd::Error> {
|
||||
base64_simd::URL_SAFE_NO_PAD.decode_to_vec(input)
|
||||
}
|
||||
|
||||
@@ -2087,7 +2087,6 @@ impl DiskAPI for LocalDisk {
|
||||
for vol in volumes {
|
||||
if let Err(e) = self.make_volume(vol).await {
|
||||
if e != DiskError::VolumeExists {
|
||||
error!("local disk make volumes failed: {e}");
|
||||
return Err(e);
|
||||
}
|
||||
}
|
||||
@@ -2109,7 +2108,6 @@ impl DiskAPI for LocalDisk {
|
||||
os::make_dir_all(&volume_dir, self.root.as_path()).await?;
|
||||
return Ok(());
|
||||
}
|
||||
error!("local disk make volume failed: {e}");
|
||||
return Err(to_volume_error(e).into());
|
||||
}
|
||||
|
||||
|
||||
@@ -301,10 +301,6 @@ impl Erasure {
|
||||
written += n;
|
||||
}
|
||||
|
||||
if ret_err.is_some() {
|
||||
return (written, ret_err);
|
||||
}
|
||||
|
||||
if written < length {
|
||||
ret_err = Some(Error::LessData.into());
|
||||
}
|
||||
|
||||
@@ -145,9 +145,7 @@ impl Erasure {
|
||||
return Err(std::io::Error::other(format!("Failed to send encoded data : {err}")));
|
||||
}
|
||||
}
|
||||
Ok(_) => {
|
||||
break;
|
||||
}
|
||||
Ok(_) => break,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -468,21 +468,15 @@ impl Erasure {
|
||||
let mut buf = vec![0u8; block_size];
|
||||
match rustfs_utils::read_full(&mut *reader, &mut buf).await {
|
||||
Ok(n) if n > 0 => {
|
||||
warn!("encode_stream_callback_async read n={}", n);
|
||||
total += n;
|
||||
let res = self.encode_data(&buf[..n]);
|
||||
on_block(res).await?
|
||||
}
|
||||
Ok(_) => {
|
||||
warn!("encode_stream_callback_async read unexpected ok");
|
||||
break;
|
||||
}
|
||||
Ok(_) => break,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::UnexpectedEof => {
|
||||
warn!("encode_stream_callback_async read unexpected eof");
|
||||
break;
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("encode_stream_callback_async read error={:?}", e);
|
||||
on_block(Err(e)).await?;
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -38,7 +38,7 @@ pub const DISK_RESERVE_FRACTION: f64 = 0.15;
|
||||
|
||||
lazy_static! {
|
||||
static ref GLOBAL_RUSTFS_PORT: OnceLock<u16> = OnceLock::new();
|
||||
static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
|
||||
static ref GLOBAL_RUSTFS_EXTERNAL_PORT: OnceLock<u16> = OnceLock::new();
|
||||
pub static ref GLOBAL_OBJECT_API: OnceLock<Arc<ECStore>> = OnceLock::new();
|
||||
pub static ref GLOBAL_LOCAL_DISK: Arc<RwLock<Vec<Option<DiskStore>>>> = Arc::new(RwLock::new(Vec::new()));
|
||||
pub static ref GLOBAL_IsErasure: RwLock<bool> = RwLock::new(false);
|
||||
@@ -51,6 +51,8 @@ lazy_static! {
|
||||
pub static ref GLOBAL_TierConfigMgr: Arc<RwLock<TierConfigMgr>> = TierConfigMgr::new();
|
||||
pub static ref GLOBAL_LifecycleSys: Arc<LifecycleSys> = LifecycleSys::new();
|
||||
pub static ref GLOBAL_EventNotifier: Arc<RwLock<EventNotifier>> = EventNotifier::new();
|
||||
//pub static ref GLOBAL_RemoteTargetTransport
|
||||
static ref globalDeploymentIDPtr: OnceLock<Uuid> = OnceLock::new();
|
||||
pub static ref GLOBAL_BOOT_TIME: OnceCell<SystemTime> = OnceCell::new();
|
||||
pub static ref GLOBAL_LocalNodeName: String = "127.0.0.1:9000".to_string();
|
||||
pub static ref GLOBAL_LocalNodeNameHex: String = rustfs_utils::crypto::hex(GLOBAL_LocalNodeName.as_bytes());
|
||||
@@ -58,22 +60,12 @@ lazy_static! {
|
||||
pub static ref GLOBAL_REGION: OnceLock<String> = OnceLock::new();
|
||||
}
|
||||
|
||||
/// Global cancellation token for background services (data scanner and auto heal)
|
||||
// Global cancellation token for background services (data scanner and auto heal)
|
||||
static GLOBAL_BACKGROUND_SERVICES_CANCEL_TOKEN: OnceLock<CancellationToken> = OnceLock::new();
|
||||
|
||||
/// Global active credentials
|
||||
static GLOBAL_ACTIVE_CRED: OnceLock<Credentials> = OnceLock::new();
|
||||
|
||||
/// Initialize the global action credentials
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `ak` - Optional access key
|
||||
/// * `sk` - Optional secret key
|
||||
///
|
||||
/// # Returns
|
||||
/// * None
|
||||
///
|
||||
pub fn init_global_action_credentials(ak: Option<String>, sk: Option<String>) {
|
||||
pub fn init_global_action_cred(ak: Option<String>, sk: Option<String>) {
|
||||
let ak = {
|
||||
if let Some(k) = ak {
|
||||
k
|
||||
@@ -99,16 +91,11 @@ pub fn init_global_action_credentials(ak: Option<String>, sk: Option<String>) {
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Get the global action credentials
|
||||
pub fn get_global_action_cred() -> Option<Credentials> {
|
||||
GLOBAL_ACTIVE_CRED.get().cloned()
|
||||
}
|
||||
|
||||
/// Get the global rustfs port
|
||||
///
|
||||
/// # Returns
|
||||
/// * `u16` - The global rustfs port
|
||||
///
|
||||
pub fn global_rustfs_port() -> u16 {
|
||||
if let Some(p) = GLOBAL_RUSTFS_PORT.get() {
|
||||
*p
|
||||
@@ -118,44 +105,36 @@ pub fn global_rustfs_port() -> u16 {
|
||||
}
|
||||
|
||||
/// Set the global rustfs port
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `value` - The port value to set globally
|
||||
///
|
||||
/// # Returns
|
||||
/// * None
|
||||
pub fn set_global_rustfs_port(value: u16) {
|
||||
GLOBAL_RUSTFS_PORT.set(value).expect("set_global_rustfs_port fail");
|
||||
}
|
||||
|
||||
/// Set the global deployment id
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `id` - The Uuid to set as the global deployment id
|
||||
///
|
||||
/// # Returns
|
||||
/// * None
|
||||
///
|
||||
/// Get the global rustfs external port
|
||||
pub fn global_rustfs_external_port() -> u16 {
|
||||
if let Some(p) = GLOBAL_RUSTFS_EXTERNAL_PORT.get() {
|
||||
*p
|
||||
} else {
|
||||
rustfs_config::DEFAULT_PORT
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the global rustfs external port
|
||||
pub fn set_global_rustfs_external_port(value: u16) {
|
||||
GLOBAL_RUSTFS_EXTERNAL_PORT
|
||||
.set(value)
|
||||
.expect("set_global_rustfs_external_port fail");
|
||||
}
|
||||
|
||||
/// Get the global rustfs port
|
||||
pub fn set_global_deployment_id(id: Uuid) {
|
||||
globalDeploymentIDPtr.set(id).unwrap();
|
||||
}
|
||||
|
||||
/// Get the global deployment id
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Option<String>` - The global deployment id as a string, if set
|
||||
///
|
||||
pub fn get_global_deployment_id() -> Option<String> {
|
||||
globalDeploymentIDPtr.get().map(|v| v.to_string())
|
||||
}
|
||||
/// Set the global endpoints
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `eps` - A vector of PoolEndpoints to set globally
|
||||
///
|
||||
/// # Returns
|
||||
/// * None
|
||||
///
|
||||
/// Get the global deployment id
|
||||
pub fn set_global_endpoints(eps: Vec<PoolEndpoints>) {
|
||||
GLOBAL_Endpoints
|
||||
.set(EndpointServerPools::from(eps))
|
||||
@@ -163,10 +142,6 @@ pub fn set_global_endpoints(eps: Vec<PoolEndpoints>) {
|
||||
}
|
||||
|
||||
/// Get the global endpoints
|
||||
///
|
||||
/// # Returns
|
||||
/// * `EndpointServerPools` - The global endpoints
|
||||
///
|
||||
pub fn get_global_endpoints() -> EndpointServerPools {
|
||||
if let Some(eps) = GLOBAL_Endpoints.get() {
|
||||
eps.clone()
|
||||
@@ -175,63 +150,29 @@ pub fn get_global_endpoints() -> EndpointServerPools {
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new object layer instance
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Option<Arc<ECStore>>` - The global object layer instance, if set
|
||||
///
|
||||
pub fn new_object_layer_fn() -> Option<Arc<ECStore>> {
|
||||
GLOBAL_OBJECT_API.get().cloned()
|
||||
}
|
||||
|
||||
/// Set the global object layer
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `o` - The ECStore instance to set globally
|
||||
///
|
||||
/// # Returns
|
||||
/// * None
|
||||
pub async fn set_object_layer(o: Arc<ECStore>) {
|
||||
GLOBAL_OBJECT_API.set(o).expect("set_object_layer fail ")
|
||||
}
|
||||
|
||||
/// Check if the setup type is distributed erasure coding
|
||||
///
|
||||
/// # Returns
|
||||
/// * `bool` - True if the setup type is distributed erasure coding, false otherwise
|
||||
///
|
||||
pub async fn is_dist_erasure() -> bool {
|
||||
let lock = GLOBAL_IsDistErasure.read().await;
|
||||
*lock
|
||||
}
|
||||
|
||||
/// Check if the setup type is erasure coding with single data center
|
||||
///
|
||||
/// # Returns
|
||||
/// * `bool` - True if the setup type is erasure coding with single data center, false otherwise
|
||||
///
|
||||
pub async fn is_erasure_sd() -> bool {
|
||||
let lock = GLOBAL_IsErasureSD.read().await;
|
||||
*lock
|
||||
}
|
||||
|
||||
/// Check if the setup type is erasure coding
|
||||
///
|
||||
/// # Returns
|
||||
/// * `bool` - True if the setup type is erasure coding, false otherwise
|
||||
///
|
||||
pub async fn is_erasure() -> bool {
|
||||
let lock = GLOBAL_IsErasure.read().await;
|
||||
*lock
|
||||
}
|
||||
|
||||
/// Update the global erasure type based on the setup type
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `setup_type` - The SetupType to update the global erasure type
|
||||
///
|
||||
/// # Returns
|
||||
/// * None
|
||||
pub async fn update_erasure_type(setup_type: SetupType) {
|
||||
let mut is_erasure = GLOBAL_IsErasure.write().await;
|
||||
*is_erasure = setup_type == SetupType::Erasure;
|
||||
@@ -257,53 +198,25 @@ pub async fn update_erasure_type(setup_type: SetupType) {
|
||||
|
||||
type TypeLocalDiskSetDrives = Vec<Vec<Vec<Option<DiskStore>>>>;
|
||||
|
||||
/// Set the global region
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `region` - The region string to set globally
|
||||
///
|
||||
/// # Returns
|
||||
/// * None
|
||||
pub fn set_global_region(region: String) {
|
||||
GLOBAL_REGION.set(region).unwrap();
|
||||
}
|
||||
|
||||
/// Get the global region
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Option<String>` - The global region string, if set
|
||||
///
|
||||
pub fn get_global_region() -> Option<String> {
|
||||
GLOBAL_REGION.get().cloned()
|
||||
}
|
||||
|
||||
/// Initialize the global background services cancellation token
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `cancel_token` - The CancellationToken instance to set globally
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Ok(())` if successful
|
||||
/// * `Err(CancellationToken)` if setting fails
|
||||
///
|
||||
pub fn init_background_services_cancel_token(cancel_token: CancellationToken) -> Result<(), CancellationToken> {
|
||||
GLOBAL_BACKGROUND_SERVICES_CANCEL_TOKEN.set(cancel_token)
|
||||
}
|
||||
|
||||
/// Get the global background services cancellation token
|
||||
///
|
||||
/// # Returns
|
||||
/// * `Option<&'static CancellationToken>` - The global cancellation token, if set
|
||||
///
|
||||
pub fn get_background_services_cancel_token() -> Option<&'static CancellationToken> {
|
||||
GLOBAL_BACKGROUND_SERVICES_CANCEL_TOKEN.get()
|
||||
}
|
||||
|
||||
/// Create and initialize the global background services cancellation token
|
||||
///
|
||||
/// # Returns
|
||||
/// * `CancellationToken` - The newly created global cancellation token
|
||||
///
|
||||
pub fn create_background_services_cancel_token() -> CancellationToken {
|
||||
let cancel_token = CancellationToken::new();
|
||||
init_background_services_cancel_token(cancel_token.clone()).expect("Background services cancel token already initialized");
|
||||
@@ -311,9 +224,6 @@ pub fn create_background_services_cancel_token() -> CancellationToken {
|
||||
}
|
||||
|
||||
/// Shutdown all background services gracefully
|
||||
///
|
||||
/// # Returns
|
||||
/// * None
|
||||
pub fn shutdown_background_services() {
|
||||
if let Some(cancel_token) = GLOBAL_BACKGROUND_SERVICES_CANCEL_TOKEN.get() {
|
||||
cancel_token.cancel();
|
||||
|
||||
@@ -44,7 +44,7 @@ mod store_init;
|
||||
pub mod store_list_objects;
|
||||
pub mod store_utils;
|
||||
|
||||
// pub mod checksum;
|
||||
pub mod checksum;
|
||||
pub mod client;
|
||||
pub mod event;
|
||||
pub mod event_notification;
|
||||
|
||||
@@ -23,7 +23,7 @@ use rustfs_common::{
|
||||
use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, RealtimeMetrics};
|
||||
use rustfs_utils::os::get_drive_stats;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{debug, info};
|
||||
use tracing::info;
|
||||
|
||||
use crate::{
|
||||
admin_server_info::get_local_server_property,
|
||||
@@ -44,7 +44,7 @@ pub struct CollectMetricsOpts {
|
||||
pub struct MetricType(u32);
|
||||
|
||||
impl MetricType {
|
||||
// Define some constants
|
||||
// 定义一些常量
|
||||
pub const NONE: MetricType = MetricType(0);
|
||||
pub const SCANNER: MetricType = MetricType(1 << 0);
|
||||
pub const DISK: MetricType = MetricType(1 << 1);
|
||||
@@ -70,18 +70,8 @@ impl MetricType {
|
||||
}
|
||||
}
|
||||
|
||||
/// Collect local metrics based on the specified types and options.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `types` - A `MetricType` specifying which types of metrics to collect.
|
||||
/// * `opts` - A reference to `CollectMetricsOpts` containing additional options for metric collection.
|
||||
///
|
||||
/// # Returns
|
||||
/// * A `RealtimeMetrics` struct containing the collected metrics.
|
||||
///
|
||||
pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts) -> RealtimeMetrics {
|
||||
debug!("collect_local_metrics");
|
||||
info!("collect_local_metrics");
|
||||
let mut real_time_metrics = RealtimeMetrics::default();
|
||||
if types.0 == MetricType::NONE.0 {
|
||||
info!("types is None, return");
|
||||
@@ -103,13 +93,13 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
|
||||
}
|
||||
|
||||
if types.contains(&MetricType::DISK) {
|
||||
debug!("start get disk metrics");
|
||||
info!("start get disk metrics");
|
||||
let mut aggr = DiskMetric {
|
||||
collected_at: Utc::now(),
|
||||
..Default::default()
|
||||
};
|
||||
for (name, disk) in collect_local_disks_metrics(&opts.disks).await.into_iter() {
|
||||
debug!("got disk metric, name: {name}, metric: {disk:?}");
|
||||
info!("got disk metric, name: {name}, metric: {disk:?}");
|
||||
real_time_metrics.by_disk.insert(name, disk.clone());
|
||||
aggr.merge(&disk);
|
||||
}
|
||||
@@ -117,7 +107,7 @@ pub async fn collect_local_metrics(types: MetricType, opts: &CollectMetricsOpts)
|
||||
}
|
||||
|
||||
if types.contains(&MetricType::SCANNER) {
|
||||
debug!("start get scanner metrics");
|
||||
info!("start get scanner metrics");
|
||||
let metrics = globalMetrics.report().await;
|
||||
real_time_metrics.aggregated.scanner = Some(metrics);
|
||||
}
|
||||
|
||||
@@ -1140,7 +1140,6 @@ impl ECStore {
|
||||
.await
|
||||
{
|
||||
if !is_err_bucket_exists(&err) {
|
||||
error!("decommission: make bucket failed: {err}");
|
||||
return Err(err);
|
||||
}
|
||||
}
|
||||
@@ -1263,8 +1262,6 @@ impl ECStore {
|
||||
parts[i] = CompletePart {
|
||||
part_num: pi.part_num,
|
||||
etag: pi.etag,
|
||||
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1292,7 +1289,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let reader = BufReader::new(rd.stream);
|
||||
let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, None, false)?;
|
||||
let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, false)?;
|
||||
let mut data = PutObjReader::new(hrd);
|
||||
|
||||
if let Err(err) = self
|
||||
|
||||
@@ -979,7 +979,6 @@ impl ECStore {
|
||||
parts[i] = CompletePart {
|
||||
part_num: pi.part_num,
|
||||
etag: pi.etag,
|
||||
..Default::default()
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1006,7 +1005,7 @@ impl ECStore {
|
||||
}
|
||||
|
||||
let reader = BufReader::new(rd.stream);
|
||||
let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, None, false)?;
|
||||
let hrd = HashReader::new(Box::new(WarpReader::new(reader)), object_info.size, object_info.size, None, false)?;
|
||||
let mut data = PutObjReader::new(hrd);
|
||||
|
||||
if let Err(err) = self
|
||||
|
||||
@@ -21,7 +21,7 @@ use crate::bucket::lifecycle::lifecycle::TRANSITION_COMPLETE;
|
||||
use crate::bucket::replication::check_replicate_delete;
|
||||
use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::client::{object_api_utils::get_raw_etag, transition_api::ReaderImpl};
|
||||
use crate::client::{object_api_utils::extract_etag, transition_api::ReaderImpl};
|
||||
use crate::disk::STORAGE_FORMAT_FILE;
|
||||
use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_read_quorum_errs, reduce_write_quorum_errs};
|
||||
use crate::disk::{
|
||||
@@ -72,13 +72,13 @@ use rustfs_filemeta::{
|
||||
};
|
||||
use rustfs_lock::fast_lock::types::LockResult;
|
||||
use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem};
|
||||
use rustfs_rio::{EtagResolvable, HashReader, HashReaderMut, TryGetIndex as _, WarpReader};
|
||||
use rustfs_rio::{EtagResolvable, HashReader, TryGetIndex as _, WarpReader};
|
||||
use rustfs_utils::http::headers::AMZ_OBJECT_TAGGING;
|
||||
use rustfs_utils::http::headers::AMZ_STORAGE_CLASS;
|
||||
use rustfs_utils::http::headers::RESERVED_METADATA_PREFIX_LOWER;
|
||||
use rustfs_utils::{
|
||||
HashAlgorithm,
|
||||
crypto::hex,
|
||||
crypto::{base64_decode, base64_encode, hex},
|
||||
path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf},
|
||||
};
|
||||
use rustfs_workers::workers::Workers;
|
||||
@@ -158,7 +158,10 @@ impl SetDisks {
|
||||
LockResult::Conflict {
|
||||
current_owner,
|
||||
current_mode,
|
||||
} => format!("{mode} lock conflicted on {bucket}/{object}: held by {current_owner} as {current_mode:?}"),
|
||||
} => format!(
|
||||
"{mode} lock conflicted on {bucket}/{object}: held by {current_owner} as {:?}",
|
||||
current_mode
|
||||
),
|
||||
LockResult::Acquired => format!("unexpected lock state while acquiring {mode} lock on {bucket}/{object}"),
|
||||
}
|
||||
}
|
||||
@@ -919,8 +922,9 @@ impl SetDisks {
|
||||
}
|
||||
|
||||
fn get_upload_id_dir(bucket: &str, object: &str, upload_id: &str) -> String {
|
||||
let upload_uuid = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(upload_id.as_bytes())
|
||||
// warn!("get_upload_id_dir upload_id {:?}", upload_id);
|
||||
|
||||
let upload_uuid = base64_decode(upload_id.as_bytes())
|
||||
.and_then(|v| {
|
||||
String::from_utf8(v).map_or(Ok(upload_id.to_owned()), |v| {
|
||||
let parts: Vec<_> = v.splitn(2, '.').collect();
|
||||
@@ -2946,7 +2950,6 @@ impl SetDisks {
|
||||
part.mod_time,
|
||||
part.actual_size,
|
||||
part.index.clone(),
|
||||
part.checksums.clone(),
|
||||
);
|
||||
if is_inline_buffer {
|
||||
if let Some(writer) = writers[index].take() {
|
||||
@@ -3525,9 +3528,9 @@ impl ObjectIO for SetDisks {
|
||||
// }
|
||||
|
||||
if object_info.size == 0 {
|
||||
// if let Some(rs) = range {
|
||||
// let _ = rs.get_offset_length(object_info.size)?;
|
||||
// }
|
||||
if let Some(rs) = range {
|
||||
let _ = rs.get_offset_length(object_info.size)?;
|
||||
}
|
||||
|
||||
let reader = GetObjectReader {
|
||||
stream: Box::new(Cursor::new(Vec::new())),
|
||||
@@ -3709,7 +3712,7 @@ impl ObjectIO for SetDisks {
|
||||
|
||||
let stream = mem::replace(
|
||||
&mut data.stream,
|
||||
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, None, false)?,
|
||||
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, false)?,
|
||||
);
|
||||
|
||||
let (reader, w_size) = match Arc::new(erasure).encode(stream, &mut writers, write_quorum).await {
|
||||
@@ -3726,12 +3729,7 @@ impl ObjectIO for SetDisks {
|
||||
// }
|
||||
|
||||
if (w_size as i64) < data.size() {
|
||||
warn!("put_object write size < data.size(), w_size={}, data.size={}", w_size, data.size());
|
||||
return Err(Error::other(format!(
|
||||
"put_object write size < data.size(), w_size={}, data.size={}",
|
||||
w_size,
|
||||
data.size()
|
||||
)));
|
||||
return Err(Error::other("put_object write size < data.size()"));
|
||||
}
|
||||
|
||||
if user_defined.contains_key(&format!("{RESERVED_METADATA_PREFIX_LOWER}compression")) {
|
||||
@@ -3758,42 +3756,31 @@ impl ObjectIO for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if fi.checksum.is_none() {
|
||||
if let Some(content_hash) = data.as_hash_reader().content_hash() {
|
||||
fi.checksum = Some(content_hash.to_bytes(&[]));
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(sc) = user_defined.get(AMZ_STORAGE_CLASS) {
|
||||
if sc == storageclass::STANDARD {
|
||||
let _ = user_defined.remove(AMZ_STORAGE_CLASS);
|
||||
}
|
||||
}
|
||||
|
||||
let mod_time = if let Some(mod_time) = opts.mod_time {
|
||||
Some(mod_time)
|
||||
} else {
|
||||
Some(OffsetDateTime::now_utc())
|
||||
};
|
||||
let now = OffsetDateTime::now_utc();
|
||||
|
||||
for (i, pfi) in parts_metadatas.iter_mut().enumerate() {
|
||||
pfi.metadata = user_defined.clone();
|
||||
for (i, fi) in parts_metadatas.iter_mut().enumerate() {
|
||||
fi.metadata = user_defined.clone();
|
||||
if is_inline_buffer {
|
||||
if let Some(writer) = writers[i].take() {
|
||||
pfi.data = Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default());
|
||||
fi.data = Some(writer.into_inline_data().map(bytes::Bytes::from).unwrap_or_default());
|
||||
}
|
||||
|
||||
pfi.set_inline_data();
|
||||
fi.set_inline_data();
|
||||
}
|
||||
|
||||
pfi.mod_time = mod_time;
|
||||
pfi.size = w_size as i64;
|
||||
pfi.versioned = opts.versioned || opts.version_suspended;
|
||||
pfi.add_object_part(1, etag.clone(), w_size, mod_time, actual_size, index_op.clone(), None);
|
||||
pfi.checksum = fi.checksum.clone();
|
||||
fi.mod_time = Some(now);
|
||||
fi.size = w_size as i64;
|
||||
fi.versioned = opts.versioned || opts.version_suspended;
|
||||
fi.add_object_part(1, etag.clone(), w_size, fi.mod_time, actual_size, index_op.clone());
|
||||
|
||||
if opts.data_movement {
|
||||
pfi.set_data_moved();
|
||||
fi.set_data_moved();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3828,8 +3815,7 @@ impl ObjectIO for SetDisks {
|
||||
|
||||
fi.replication_state_internal = Some(opts.put_replication_state());
|
||||
|
||||
fi.is_latest = true;
|
||||
|
||||
// TODO: version support
|
||||
Ok(ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended))
|
||||
}
|
||||
}
|
||||
@@ -4444,6 +4430,8 @@ impl StorageAPI for SetDisks {
|
||||
.await
|
||||
.map_err(|e| to_object_err(e, vec![bucket, object]))?;
|
||||
|
||||
// warn!("get object_info fi {:?}", &fi);
|
||||
|
||||
let oi = ObjectInfo::from_file_info(&fi, bucket, object, opts.versioned || opts.version_suspended);
|
||||
|
||||
Ok(oi)
|
||||
@@ -4593,7 +4581,7 @@ impl StorageAPI for SetDisks {
|
||||
}*/
|
||||
// Normalize ETags by removing quotes before comparison (PR #592 compatibility)
|
||||
let transition_etag = rustfs_utils::path::trim_etag(&opts.transition.etag);
|
||||
let stored_etag = rustfs_utils::path::trim_etag(&get_raw_etag(&fi.metadata));
|
||||
let stored_etag = rustfs_utils::path::trim_etag(&extract_etag(&fi.metadata));
|
||||
if !opts.mod_time.expect("err").unix_timestamp() == fi.mod_time.as_ref().expect("err").unix_timestamp()
|
||||
|| transition_etag != stored_etag
|
||||
{
|
||||
@@ -4771,11 +4759,6 @@ impl StorageAPI for SetDisks {
|
||||
uploaded_parts.push(CompletePart {
|
||||
part_num: p_info.part_num,
|
||||
etag: p_info.etag,
|
||||
checksum_crc32: None,
|
||||
checksum_crc32c: None,
|
||||
checksum_sha1: None,
|
||||
checksum_sha256: None,
|
||||
checksum_crc64nvme: None,
|
||||
});
|
||||
}
|
||||
if let Err(err) = self.complete_multipart_upload(bucket, object, &res.upload_id, uploaded_parts, &ObjectOptions {
|
||||
@@ -4851,24 +4834,64 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
let write_quorum = fi.write_quorum(self.default_write_quorum());
|
||||
|
||||
if let Some(checksum) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM)
|
||||
&& !checksum.is_empty()
|
||||
&& data
|
||||
.as_hash_reader()
|
||||
.content_crc_type()
|
||||
.is_none_or(|v| v.to_string() != *checksum)
|
||||
{
|
||||
return Err(Error::other(format!("checksum mismatch: {checksum}")));
|
||||
}
|
||||
|
||||
let disks = self.disks.read().await.clone();
|
||||
let disks = self.disks.read().await;
|
||||
|
||||
let disks = disks.clone();
|
||||
let shuffle_disks = Self::shuffle_disks(&disks, &fi.erasure.distribution);
|
||||
|
||||
let part_suffix = format!("part.{part_id}");
|
||||
let tmp_part = format!("{}x{}", Uuid::new_v4(), OffsetDateTime::now_utc().unix_timestamp());
|
||||
let tmp_part_path = Arc::new(format!("{tmp_part}/{part_suffix}"));
|
||||
|
||||
// let mut writers = Vec::with_capacity(disks.len());
|
||||
// let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
|
||||
// let shared_size = erasure.shard_size(erasure.block_size);
|
||||
|
||||
// let futures = disks.iter().map(|disk| {
|
||||
// let disk = disk.clone();
|
||||
// let tmp_part_path = tmp_part_path.clone();
|
||||
// tokio::spawn(async move {
|
||||
// if let Some(disk) = disk {
|
||||
// // let writer = disk.append_file(RUSTFS_META_TMP_BUCKET, &tmp_part_path).await?;
|
||||
// // let filewriter = disk
|
||||
// // .create_file("", RUSTFS_META_TMP_BUCKET, &tmp_part_path, data.content_length)
|
||||
// // .await?;
|
||||
// match new_bitrot_filewriter(
|
||||
// disk.clone(),
|
||||
// RUSTFS_META_TMP_BUCKET,
|
||||
// &tmp_part_path,
|
||||
// false,
|
||||
// DEFAULT_BITROT_ALGO,
|
||||
// shared_size,
|
||||
// )
|
||||
// .await
|
||||
// {
|
||||
// Ok(writer) => Ok(Some(writer)),
|
||||
// Err(e) => Err(e),
|
||||
// }
|
||||
// } else {
|
||||
// Ok(None)
|
||||
// }
|
||||
// })
|
||||
// });
|
||||
// for x in join_all(futures).await {
|
||||
// let x = x??;
|
||||
// writers.push(x);
|
||||
// }
|
||||
|
||||
// let erasure = Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
|
||||
|
||||
// let stream = replace(&mut data.stream, Box::new(empty()));
|
||||
// let etag_stream = EtagReader::new(stream);
|
||||
|
||||
// let (w_size, mut etag) = Arc::new(erasure)
|
||||
// .encode(etag_stream, &mut writers, data.content_length, write_quorum)
|
||||
// .await?;
|
||||
|
||||
// if let Err(err) = close_bitrot_writers(&mut writers).await {
|
||||
// error!("close_bitrot_writers err {:?}", err);
|
||||
// }
|
||||
|
||||
let erasure = erasure_coding::Erasure::new(fi.erasure.data_blocks, fi.erasure.parity_blocks, fi.erasure.block_size);
|
||||
|
||||
let mut writers = Vec::with_capacity(shuffle_disks.len());
|
||||
@@ -4921,7 +4944,7 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
let stream = mem::replace(
|
||||
&mut data.stream,
|
||||
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, None, false)?,
|
||||
HashReader::new(Box::new(WarpReader::new(Cursor::new(Vec::new()))), 0, 0, None, false)?,
|
||||
);
|
||||
|
||||
let (reader, w_size) = Arc::new(erasure).encode(stream, &mut writers, write_quorum).await?; // TODO: 出错，删除临时目录
|
||||
@@ -4929,12 +4952,7 @@ impl StorageAPI for SetDisks {
|
||||
let _ = mem::replace(&mut data.stream, reader);
|
||||
|
||||
if (w_size as i64) < data.size() {
|
||||
warn!("put_object_part write size < data.size(), w_size={}, data.size={}", w_size, data.size());
|
||||
return Err(Error::other(format!(
|
||||
"put_object_part write size < data.size(), w_size={}, data.size={}",
|
||||
w_size,
|
||||
data.size()
|
||||
)));
|
||||
return Err(Error::other("put_object_part write size < data.size()"));
|
||||
}
|
||||
|
||||
let index_op = data.stream.try_get_index().map(|v| v.clone().into_vec());
|
||||
@@ -5209,8 +5227,7 @@ impl StorageAPI for SetDisks {
|
||||
uploads.push(MultipartInfo {
|
||||
bucket: bucket.to_owned(),
|
||||
object: object.to_owned(),
|
||||
upload_id: base64_simd::URL_SAFE_NO_PAD
|
||||
.encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes()),
|
||||
upload_id: base64_encode(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_id).as_bytes()),
|
||||
initiated: Some(start_time),
|
||||
..Default::default()
|
||||
});
|
||||
@@ -5331,14 +5348,6 @@ impl StorageAPI for SetDisks {
|
||||
}
|
||||
}
|
||||
|
||||
if let Some(checksum) = &opts.want_checksum {
|
||||
user_defined.insert(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM.to_string(), checksum.checksum_type.to_string());
|
||||
user_defined.insert(
|
||||
rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE.to_string(),
|
||||
checksum.checksum_type.obj_type().to_string(),
|
||||
);
|
||||
}
|
||||
|
||||
let (shuffle_disks, mut parts_metadatas) = Self::shuffle_disks_and_parts_metadata(&disks, &parts_metadata, &fi);
|
||||
|
||||
let mod_time = opts.mod_time.unwrap_or(OffsetDateTime::now_utc());
|
||||
@@ -5353,8 +5362,7 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
let upload_uuid = format!("{}x{}", Uuid::new_v4(), mod_time.unix_timestamp_nanos());
|
||||
|
||||
let upload_id = base64_simd::URL_SAFE_NO_PAD
|
||||
.encode_to_string(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_uuid).as_bytes());
|
||||
let upload_id = base64_encode(format!("{}.{}", get_global_deployment_id().unwrap_or_default(), upload_uuid).as_bytes());
|
||||
|
||||
let upload_path = Self::get_upload_id_dir(bucket, object, upload_uuid.as_str());
|
||||
|
||||
@@ -5371,11 +5379,7 @@ impl StorageAPI for SetDisks {
|
||||
|
||||
// evalDisks
|
||||
|
||||
Ok(MultipartUploadResult {
|
||||
upload_id,
|
||||
checksum_algo: user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM).cloned(),
|
||||
checksum_type: user_defined.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE).cloned(),
|
||||
})
|
||||
Ok(MultipartUploadResult { upload_id })
|
||||
}
|
||||
|
||||
#[tracing::instrument(skip(self))]
|
||||
@@ -5463,25 +5467,6 @@ impl StorageAPI for SetDisks {
|
||||
return Err(Error::other("part result number err"));
|
||||
}
|
||||
|
||||
if let Some(cs) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM) {
|
||||
let Some(checksum_type) = fi.metadata.get(rustfs_rio::RUSTFS_MULTIPART_CHECKSUM_TYPE) else {
|
||||
return Err(Error::other("checksum type not found"));
|
||||
};
|
||||
|
||||
if opts.want_checksum.is_some()
|
||||
&& !opts.want_checksum.as_ref().is_some_and(|v| {
|
||||
v.checksum_type
|
||||
.is(rustfs_rio::ChecksumType::from_string_with_obj_type(cs, checksum_type))
|
||||
})
|
||||
{
|
||||
return Err(Error::other(format!(
|
||||
"checksum type mismatch, got {:?}, want {:?}",
|
||||
opts.want_checksum.as_ref().unwrap(),
|
||||
rustfs_rio::ChecksumType::from_string_with_obj_type(cs, checksum_type)
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
for (i, part) in object_parts.iter().enumerate() {
|
||||
if let Some(err) = &part.error {
|
||||
error!("complete_multipart_upload part error: {:?}", &err);
|
||||
@@ -5502,7 +5487,6 @@ impl StorageAPI for SetDisks {
|
||||
part.mod_time,
|
||||
part.actual_size,
|
||||
part.index.clone(),
|
||||
part.checksums.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
@@ -6438,20 +6422,10 @@ mod tests {
|
||||
CompletePart {
|
||||
part_num: 1,
|
||||
etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()),
|
||||
checksum_crc32: None,
|
||||
checksum_crc32c: None,
|
||||
checksum_sha1: None,
|
||||
checksum_sha256: None,
|
||||
checksum_crc64nvme: None,
|
||||
},
|
||||
CompletePart {
|
||||
part_num: 2,
|
||||
etag: Some("098f6bcd4621d373cade4e832627b4f6".to_string()),
|
||||
checksum_crc32: None,
|
||||
checksum_crc32c: None,
|
||||
checksum_sha1: None,
|
||||
checksum_sha256: None,
|
||||
checksum_crc64nvme: None,
|
||||
},
|
||||
];
|
||||
|
||||
@@ -6468,11 +6442,6 @@ mod tests {
|
||||
let single_part = vec![CompletePart {
|
||||
part_num: 1,
|
||||
etag: Some("d41d8cd98f00b204e9800998ecf8427e".to_string()),
|
||||
checksum_crc32: None,
|
||||
checksum_crc32c: None,
|
||||
checksum_sha1: None,
|
||||
checksum_sha256: None,
|
||||
checksum_crc64nvme: None,
|
||||
}];
|
||||
let single_result = get_complete_multipart_md5(&single_part);
|
||||
assert!(single_result.ends_with("-1"));
|
||||
|
||||
@@ -59,6 +59,7 @@ use rustfs_common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_
|
||||
use rustfs_common::heal_channel::{HealItemType, HealOpts};
|
||||
use rustfs_filemeta::FileInfo;
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_utils::crypto::base64_decode;
|
||||
use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf};
|
||||
use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration};
|
||||
use std::cmp::Ordering;
|
||||
@@ -1230,7 +1231,6 @@ impl StorageAPI for ECStore {
|
||||
if let Err(err) = self.peer_sys.make_bucket(bucket, opts).await {
|
||||
let err = to_object_err(err.into(), vec![bucket]);
|
||||
if !is_err_bucket_exists(&err) {
|
||||
error!("make bucket failed: {err}");
|
||||
let _ = self
|
||||
.delete_bucket(
|
||||
bucket,
|
||||
@@ -2421,7 +2421,7 @@ fn check_list_multipart_args(
|
||||
}
|
||||
}
|
||||
|
||||
if let Err(_e) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id_marker.as_bytes()) {
|
||||
if let Err(_e) = base64_decode(upload_id_marker.as_bytes()) {
|
||||
return Err(StorageError::MalformedUploadID(upload_id_marker.to_owned()));
|
||||
}
|
||||
}
|
||||
@@ -2448,7 +2448,7 @@ fn check_new_multipart_args(bucket: &str, object: &str) -> Result<()> {
|
||||
}
|
||||
|
||||
fn check_multipart_object_args(bucket: &str, object: &str, upload_id: &str) -> Result<()> {
|
||||
if let Err(e) = base64_simd::URL_SAFE_NO_PAD.decode_to_vec(upload_id.as_bytes()) {
|
||||
if let Err(e) = base64_decode(upload_id.as_bytes()) {
|
||||
return Err(StorageError::MalformedUploadID(format!("{bucket}/{object}-{upload_id},err:{e}")));
|
||||
};
|
||||
check_object_args(bucket, object)
|
||||
|
||||
@@ -13,6 +13,9 @@
|
||||
// limitations under the License.
|
||||
|
||||
use crate::bucket::metadata_sys::get_versioning_config;
|
||||
use crate::bucket::replication::REPLICATION_RESET;
|
||||
use crate::bucket::replication::REPLICATION_STATUS;
|
||||
use crate::bucket::replication::{ReplicateDecision, replication_statuses_map, version_purge_statuses_map};
|
||||
use crate::bucket::versioning::VersioningApi as _;
|
||||
use crate::disk::DiskStore;
|
||||
use crate::error::{Error, Result};
|
||||
@@ -22,15 +25,12 @@ use crate::{
|
||||
bucket::lifecycle::lifecycle::ExpirationOptions,
|
||||
bucket::lifecycle::{bucket_lifecycle_ops::TransitionedObject, lifecycle::TransitionOptions},
|
||||
};
|
||||
use bytes::Bytes;
|
||||
use http::{HeaderMap, HeaderValue};
|
||||
use rustfs_common::heal_channel::HealOpts;
|
||||
use rustfs_filemeta::{
|
||||
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, REPLICATION_RESET, REPLICATION_STATUS, ReplicateDecision, ReplicationState,
|
||||
ReplicationStatusType, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
|
||||
FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, ReplicationState, ReplicationStatusType, VersionPurgeStatusType,
|
||||
};
|
||||
use rustfs_madmin::heal_commands::HealResultItem;
|
||||
use rustfs_rio::Checksum;
|
||||
use rustfs_rio::{DecompressReader, HashReader, LimitReader, WarpReader};
|
||||
use rustfs_utils::CompressionAlgorithm;
|
||||
use rustfs_utils::http::headers::{AMZ_OBJECT_TAGGING, RESERVED_METADATA_PREFIX_LOWER};
|
||||
@@ -92,28 +92,11 @@ impl PutObjReader {
|
||||
PutObjReader { stream }
|
||||
}
|
||||
|
||||
pub fn as_hash_reader(&self) -> &HashReader {
|
||||
&self.stream
|
||||
}
|
||||
|
||||
pub fn from_vec(data: Vec<u8>) -> Self {
|
||||
use sha2::{Digest, Sha256};
|
||||
let content_length = data.len() as i64;
|
||||
let sha256hex = if content_length > 0 {
|
||||
Some(hex_simd::encode_to_string(Sha256::digest(&data), hex_simd::AsciiCase::Lower))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
PutObjReader {
|
||||
stream: HashReader::new(
|
||||
Box::new(WarpReader::new(Cursor::new(data))),
|
||||
content_length,
|
||||
content_length,
|
||||
None,
|
||||
sha256hex,
|
||||
false,
|
||||
)
|
||||
.unwrap(),
|
||||
stream: HashReader::new(Box::new(WarpReader::new(Cursor::new(data))), content_length, content_length, None, false)
|
||||
.unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -264,16 +247,11 @@ impl HTTPRangeSpec {
|
||||
return None;
|
||||
}
|
||||
|
||||
if part_number == 0 || part_number > oi.parts.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let mut start = 0_i64;
|
||||
let mut end = -1_i64;
|
||||
for i in 0..part_number {
|
||||
let part = &oi.parts[i];
|
||||
let mut start = 0i64;
|
||||
let mut end = -1i64;
|
||||
for i in 0..oi.parts.len().min(part_number) {
|
||||
start = end + 1;
|
||||
end = start + (part.size as i64) - 1;
|
||||
end = start + (oi.parts[i].size as i64) - 1
|
||||
}
|
||||
|
||||
Some(HTTPRangeSpec {
|
||||
@@ -288,14 +266,8 @@ impl HTTPRangeSpec {
|
||||
|
||||
let mut start = self.start;
|
||||
if self.is_suffix_length {
|
||||
let suffix_len = if self.start < 0 {
|
||||
self.start
|
||||
.checked_neg()
|
||||
.ok_or_else(|| Error::other("range value invalid: suffix length overflow"))?
|
||||
} else {
|
||||
self.start
|
||||
};
|
||||
start = res_size - suffix_len;
|
||||
start = res_size + self.start;
|
||||
|
||||
if start < 0 {
|
||||
start = 0;
|
||||
}
|
||||
@@ -308,13 +280,7 @@ impl HTTPRangeSpec {
|
||||
}
|
||||
|
||||
if self.is_suffix_length {
|
||||
let specified_len = if self.start < 0 {
|
||||
self.start
|
||||
.checked_neg()
|
||||
.ok_or_else(|| Error::other("range value invalid: suffix length overflow"))?
|
||||
} else {
|
||||
self.start
|
||||
};
|
||||
let specified_len = self.start; // 假设 h.start 是一个 i64 类型
|
||||
let mut range_length = specified_len;
|
||||
|
||||
if specified_len > res_size {
|
||||
@@ -391,8 +357,6 @@ pub struct ObjectOptions {
|
||||
pub lifecycle_audit_event: LcAuditEvent,
|
||||
|
||||
pub eval_metadata: Option<HashMap<String, String>>,
|
||||
|
||||
pub want_checksum: Option<Checksum>,
|
||||
}
|
||||
|
||||
impl ObjectOptions {
|
||||
@@ -475,8 +439,6 @@ pub struct BucketInfo {
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub struct MultipartUploadResult {
|
||||
pub upload_id: String,
|
||||
pub checksum_algo: Option<String>,
|
||||
pub checksum_type: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Default, Clone)]
|
||||
@@ -492,24 +454,13 @@ pub struct PartInfo {
|
||||
pub struct CompletePart {
|
||||
pub part_num: usize,
|
||||
pub etag: Option<String>,
|
||||
// pub size: Option<usize>,
|
||||
pub checksum_crc32: Option<String>,
|
||||
pub checksum_crc32c: Option<String>,
|
||||
pub checksum_sha1: Option<String>,
|
||||
pub checksum_sha256: Option<String>,
|
||||
pub checksum_crc64nvme: Option<String>,
|
||||
}
|
||||
|
||||
impl From<s3s::dto::CompletedPart> for CompletePart {
|
||||
fn from(value: s3s::dto::CompletedPart) -> Self {
|
||||
Self {
|
||||
part_num: value.part_number.unwrap_or_default() as usize,
|
||||
etag: value.e_tag.map(|v| v.value().to_owned()),
|
||||
checksum_crc32: value.checksum_crc32,
|
||||
checksum_crc32c: value.checksum_crc32c,
|
||||
checksum_sha1: value.checksum_sha1,
|
||||
checksum_sha256: value.checksum_sha256,
|
||||
checksum_crc64nvme: value.checksum_crc64nvme,
|
||||
etag: value.e_tag,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -549,7 +500,7 @@ pub struct ObjectInfo {
|
||||
pub version_purge_status_internal: Option<String>,
|
||||
pub version_purge_status: VersionPurgeStatusType,
|
||||
pub replication_decision: String,
|
||||
pub checksum: Option<Bytes>,
|
||||
pub checksum: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Clone for ObjectInfo {
|
||||
@@ -586,7 +537,7 @@ impl Clone for ObjectInfo {
|
||||
version_purge_status_internal: self.version_purge_status_internal.clone(),
|
||||
version_purge_status: self.version_purge_status.clone(),
|
||||
replication_decision: self.replication_decision.clone(),
|
||||
checksum: self.checksum.clone(),
|
||||
checksum: Default::default(),
|
||||
expires: self.expires,
|
||||
}
|
||||
}
|
||||
@@ -726,7 +677,6 @@ impl ObjectInfo {
|
||||
inlined,
|
||||
user_defined: metadata,
|
||||
transitioned_object,
|
||||
checksum: fi.checksum.clone(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -917,23 +867,6 @@ impl ObjectInfo {
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn decrypt_checksums(&self, part: usize, _headers: &HeaderMap) -> Result<(HashMap<String, String>, bool)> {
|
||||
if part > 0 {
|
||||
if let Some(checksums) = self.parts.iter().find(|p| p.number == part).and_then(|p| p.checksums.clone()) {
|
||||
return Ok((checksums, true));
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: decrypt checksums
|
||||
|
||||
if let Some(data) = &self.checksum {
|
||||
let (checksums, is_multipart) = rustfs_rio::read_checksums(data.as_ref(), 0);
|
||||
return Ok((checksums, is_multipart));
|
||||
}
|
||||
|
||||
Ok((HashMap::new(), false))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
@@ -1628,83 +1561,6 @@ mod tests {
|
||||
assert_eq!(length, 10); // end - start + 1 = 14 - 5 + 1 = 10
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_range_spec_suffix_positive_start() {
|
||||
let range_spec = HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: 5,
|
||||
end: -1,
|
||||
};
|
||||
|
||||
let (offset, length) = range_spec.get_offset_length(20).unwrap();
|
||||
assert_eq!(offset, 15);
|
||||
assert_eq!(length, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_range_spec_suffix_negative_start() {
|
||||
let range_spec = HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: -5,
|
||||
end: -1,
|
||||
};
|
||||
|
||||
let (offset, length) = range_spec.get_offset_length(20).unwrap();
|
||||
assert_eq!(offset, 15);
|
||||
assert_eq!(length, 5);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_range_spec_suffix_exceeds_object() {
|
||||
let range_spec = HTTPRangeSpec {
|
||||
is_suffix_length: true,
|
||||
start: 50,
|
||||
end: -1,
|
||||
};
|
||||
|
||||
let (offset, length) = range_spec.get_offset_length(20).unwrap();
|
||||
assert_eq!(offset, 0);
|
||||
assert_eq!(length, 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_http_range_spec_from_object_info_valid_and_invalid_parts() {
|
||||
let object_info = ObjectInfo {
|
||||
size: 300,
|
||||
parts: vec![
|
||||
ObjectPartInfo {
|
||||
etag: String::new(),
|
||||
number: 1,
|
||||
size: 100,
|
||||
actual_size: 100,
|
||||
..Default::default()
|
||||
},
|
||||
ObjectPartInfo {
|
||||
etag: String::new(),
|
||||
number: 2,
|
||||
size: 100,
|
||||
actual_size: 100,
|
||||
..Default::default()
|
||||
},
|
||||
ObjectPartInfo {
|
||||
etag: String::new(),
|
||||
number: 3,
|
||||
size: 100,
|
||||
actual_size: 100,
|
||||
..Default::default()
|
||||
},
|
||||
],
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let spec = HTTPRangeSpec::from_object_info(&object_info, 2).unwrap();
|
||||
assert_eq!(spec.start, 100);
|
||||
assert_eq!(spec.end, 199);
|
||||
|
||||
assert!(HTTPRangeSpec::from_object_info(&object_info, 0).is_none());
|
||||
assert!(HTTPRangeSpec::from_object_info(&object_info, 4).is_none());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_ranged_decompress_reader_zero_length() {
|
||||
let original_data = b"Hello, World!";
|
||||
|
||||
@@ -40,8 +40,6 @@ byteorder = { workspace = true }
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
s3s.workspace = true
|
||||
lazy_static.workspace = true
|
||||
regex.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
criterion = { workspace = true }
|
||||
|
||||
@@ -284,7 +284,6 @@ impl FileInfo {
|
||||
Ok(t)
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn add_object_part(
|
||||
&mut self,
|
||||
num: usize,
|
||||
@@ -293,7 +292,6 @@ impl FileInfo {
|
||||
mod_time: Option<OffsetDateTime>,
|
||||
actual_size: i64,
|
||||
index: Option<Bytes>,
|
||||
checksums: Option<HashMap<String, String>>,
|
||||
) {
|
||||
let part = ObjectPartInfo {
|
||||
etag,
|
||||
@@ -302,7 +300,7 @@ impl FileInfo {
|
||||
mod_time,
|
||||
actual_size,
|
||||
index,
|
||||
checksums,
|
||||
checksums: None,
|
||||
error: None,
|
||||
};
|
||||
|
||||
|
||||
@@ -15,12 +15,9 @@
|
||||
use crate::error::{Error, Result};
|
||||
use crate::fileinfo::{ErasureAlgo, ErasureInfo, FileInfo, FileInfoVersions, ObjectPartInfo, RawFileInfo};
|
||||
use crate::filemeta_inline::InlineData;
|
||||
use crate::{
|
||||
ReplicationState, ReplicationStatusType, VersionPurgeStatusType, replication_statuses_map, version_purge_statuses_map,
|
||||
};
|
||||
use crate::{ReplicationStatusType, VersionPurgeStatusType};
|
||||
use byteorder::ByteOrder;
|
||||
use bytes::Bytes;
|
||||
use rustfs_utils::http::AMZ_BUCKET_REPLICATION_STATUS;
|
||||
use rustfs_utils::http::headers::{
|
||||
self, AMZ_META_UNENCRYPTED_CONTENT_LENGTH, AMZ_META_UNENCRYPTED_CONTENT_MD5, AMZ_STORAGE_CLASS, RESERVED_METADATA_PREFIX,
|
||||
RESERVED_METADATA_PREFIX_LOWER, VERSION_PURGE_STATUS_KEY,
|
||||
@@ -33,7 +30,6 @@ use std::hash::Hasher;
|
||||
use std::io::{Read, Write};
|
||||
use std::{collections::HashMap, io::Cursor};
|
||||
use time::OffsetDateTime;
|
||||
use time::format_description::well_known::Rfc3339;
|
||||
use tokio::io::AsyncRead;
|
||||
use tracing::error;
|
||||
use uuid::Uuid;
|
||||
@@ -1746,25 +1742,7 @@ impl MetaObject {
|
||||
}
|
||||
}
|
||||
|
||||
let replication_state_internal = get_internal_replication_state(&metadata);
|
||||
|
||||
let mut deleted = false;
|
||||
|
||||
if let Some(v) = replication_state_internal.as_ref() {
|
||||
if !v.composite_version_purge_status().is_empty() {
|
||||
deleted = true;
|
||||
}
|
||||
|
||||
let st = v.composite_replication_status();
|
||||
if !st.is_empty() {
|
||||
metadata.insert(AMZ_BUCKET_REPLICATION_STATUS.to_string(), st.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let checksum = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}crc").as_str())
|
||||
.map(|v| Bytes::from(v.clone()));
|
||||
// todo: ReplicationState,Delete
|
||||
|
||||
let erasure = ErasureInfo {
|
||||
algorithm: self.erasure_algorithm.to_string(),
|
||||
@@ -1776,26 +1754,6 @@ impl MetaObject {
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
let transition_status = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}").as_str())
|
||||
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||
.unwrap_or_default();
|
||||
let transitioned_objname = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}").as_str())
|
||||
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||
.unwrap_or_default();
|
||||
let transition_version_id = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}").as_str())
|
||||
.map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default());
|
||||
let transition_tier = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}").as_str())
|
||||
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
FileInfo {
|
||||
version_id,
|
||||
erasure,
|
||||
@@ -1806,13 +1764,6 @@ impl MetaObject {
|
||||
volume: volume.to_string(),
|
||||
parts,
|
||||
metadata,
|
||||
replication_state_internal,
|
||||
deleted,
|
||||
checksum,
|
||||
transition_status,
|
||||
transitioned_objname,
|
||||
transition_version_id,
|
||||
transition_tier,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -1953,38 +1904,6 @@ impl From<FileInfo> for MetaObject {
|
||||
}
|
||||
}
|
||||
|
||||
if !value.transition_status.is_empty() {
|
||||
meta_sys.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_STATUS}"),
|
||||
value.transition_status.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
|
||||
if !value.transitioned_objname.is_empty() {
|
||||
meta_sys.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}"),
|
||||
value.transitioned_objname.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(vid) = &value.transition_version_id {
|
||||
meta_sys.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}"),
|
||||
vid.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
|
||||
if !value.transition_tier.is_empty() {
|
||||
meta_sys.insert(
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}"),
|
||||
value.transition_tier.as_bytes().to_vec(),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(content_hash) = value.checksum {
|
||||
meta_sys.insert(format!("{RESERVED_METADATA_PREFIX_LOWER}crc"), content_hash.to_vec());
|
||||
}
|
||||
|
||||
Self {
|
||||
version_id: value.version_id,
|
||||
data_dir: value.data_dir,
|
||||
@@ -2008,50 +1927,6 @@ impl From<FileInfo> for MetaObject {
|
||||
}
|
||||
}
|
||||
|
||||
fn get_internal_replication_state(metadata: &HashMap<String, String>) -> Option<ReplicationState> {
|
||||
let mut rs = ReplicationState::default();
|
||||
let mut has = false;
|
||||
|
||||
for (k, v) in metadata.iter() {
|
||||
if k == VERSION_PURGE_STATUS_KEY {
|
||||
rs.version_purge_status_internal = Some(v.clone());
|
||||
rs.purge_targets = version_purge_statuses_map(v.as_str());
|
||||
has = true;
|
||||
continue;
|
||||
}
|
||||
|
||||
if let Some(sub_key) = k.strip_prefix(RESERVED_METADATA_PREFIX_LOWER) {
|
||||
match sub_key {
|
||||
"replica-timestamp" => {
|
||||
has = true;
|
||||
rs.replica_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH));
|
||||
}
|
||||
"replica-status" => {
|
||||
has = true;
|
||||
rs.replica_status = ReplicationStatusType::from(v.as_str());
|
||||
}
|
||||
"replication-timestamp" => {
|
||||
has = true;
|
||||
rs.replication_timestamp = Some(OffsetDateTime::parse(v, &Rfc3339).unwrap_or(OffsetDateTime::UNIX_EPOCH))
|
||||
}
|
||||
"replication-status" => {
|
||||
has = true;
|
||||
rs.replication_status_internal = Some(v.clone());
|
||||
rs.targets = replication_statuses_map(v.as_str());
|
||||
}
|
||||
_ => {
|
||||
if let Some(arn) = sub_key.strip_prefix("replication-reset-") {
|
||||
has = true;
|
||||
rs.reset_statuses_map.insert(arn.to_string(), v.clone());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if has { Some(rs) } else { None }
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)]
|
||||
pub struct MetaDeleteMarker {
|
||||
#[serde(rename = "ID")]
|
||||
@@ -2064,51 +1939,24 @@ pub struct MetaDeleteMarker {
|
||||
|
||||
impl MetaDeleteMarker {
|
||||
pub fn free_version(&self) -> bool {
|
||||
self.meta_sys
|
||||
.contains_key(format!("{RESERVED_METADATA_PREFIX_LOWER}{FREE_VERSION}").as_str())
|
||||
self.meta_sys.contains_key(FREE_VERSION_META_HEADER)
|
||||
}
|
||||
|
||||
pub fn into_fileinfo(&self, volume: &str, path: &str, _all_parts: bool) -> FileInfo {
|
||||
let metadata = self
|
||||
.meta_sys
|
||||
.clone()
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k, String::from_utf8_lossy(&v).to_string()))
|
||||
.collect();
|
||||
let replication_state_internal = get_internal_replication_state(&metadata);
|
||||
let metadata = self.meta_sys.clone();
|
||||
|
||||
let mut fi = FileInfo {
|
||||
FileInfo {
|
||||
version_id: self.version_id.filter(|&vid| !vid.is_nil()),
|
||||
name: path.to_string(),
|
||||
volume: volume.to_string(),
|
||||
deleted: true,
|
||||
mod_time: self.mod_time,
|
||||
metadata,
|
||||
replication_state_internal,
|
||||
metadata: metadata
|
||||
.into_iter()
|
||||
.map(|(k, v)| (k, String::from_utf8_lossy(&v).to_string()))
|
||||
.collect(),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
if self.free_version() {
|
||||
fi.set_tier_free_version();
|
||||
fi.transition_tier = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITION_TIER}").as_str())
|
||||
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
fi.transitioned_objname = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_OBJECTNAME}").as_str())
|
||||
.map(|v| String::from_utf8_lossy(v).to_string())
|
||||
.unwrap_or_default();
|
||||
|
||||
fi.transition_version_id = self
|
||||
.meta_sys
|
||||
.get(format!("{RESERVED_METADATA_PREFIX_LOWER}{TRANSITIONED_VERSION_ID}").as_str())
|
||||
.map(|v| Uuid::from_slice(v.as_slice()).unwrap_or_default());
|
||||
}
|
||||
|
||||
fi
|
||||
}
|
||||
|
||||
pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result<u64> {
|
||||
@@ -2312,6 +2160,8 @@ pub enum Flags {
|
||||
InlineData = 1 << 2,
|
||||
}
|
||||
|
||||
const FREE_VERSION_META_HEADER: &str = "free-version";
|
||||
|
||||
// mergeXLV2Versions
|
||||
pub fn merge_file_meta_versions(
|
||||
mut quorum: usize,
|
||||
|
||||
@@ -1,36 +1,8 @@
|
||||
use bytes::Bytes;
|
||||
use core::fmt;
|
||||
use regex::Regex;
|
||||
use rustfs_utils::http::RESERVED_METADATA_PREFIX_LOWER;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::any::Any;
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
use time::OffsetDateTime;
|
||||
use uuid::Uuid;
|
||||
|
||||
pub const REPLICATION_RESET: &str = "replication-reset";
|
||||
pub const REPLICATION_STATUS: &str = "replication-status";
|
||||
|
||||
// ReplicateQueued - replication being queued trail
|
||||
pub const REPLICATE_QUEUED: &str = "replicate:queue";
|
||||
|
||||
// ReplicateExisting - audit trail for existing objects replication
|
||||
pub const REPLICATE_EXISTING: &str = "replicate:existing";
|
||||
// ReplicateExistingDelete - audit trail for delete replication triggered for existing delete markers
|
||||
pub const REPLICATE_EXISTING_DELETE: &str = "replicate:existing:delete";
|
||||
|
||||
// ReplicateMRF - audit trail for replication from Most Recent Failures (MRF) queue
|
||||
pub const REPLICATE_MRF: &str = "replicate:mrf";
|
||||
// ReplicateIncoming - audit trail of inline replication
|
||||
pub const REPLICATE_INCOMING: &str = "replicate:incoming";
|
||||
// ReplicateIncomingDelete - audit trail of inline replication of deletes.
|
||||
pub const REPLICATE_INCOMING_DELETE: &str = "replicate:incoming:delete";
|
||||
|
||||
// ReplicateHeal - audit trail for healing of failed/pending replications
|
||||
pub const REPLICATE_HEAL: &str = "replicate:heal";
|
||||
// ReplicateHealDelete - audit trail of healing of failed/pending delete replications.
|
||||
pub const REPLICATE_HEAL_DELETE: &str = "replicate:heal:delete";
|
||||
|
||||
/// StatusType of Replication for x-amz-replication-status header
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize, Default, Hash)]
|
||||
@@ -520,371 +492,3 @@ impl ReplicatedInfos {
|
||||
ReplicationAction::None
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize, Deserialize, Debug)]
|
||||
pub struct MrfReplicateEntry {
|
||||
#[serde(rename = "bucket")]
|
||||
pub bucket: String,
|
||||
|
||||
#[serde(rename = "object")]
|
||||
pub object: String,
|
||||
|
||||
#[serde(skip_serializing, skip_deserializing)]
|
||||
pub version_id: Option<Uuid>,
|
||||
|
||||
#[serde(rename = "retryCount")]
|
||||
pub retry_count: i32,
|
||||
|
||||
#[serde(skip_serializing, skip_deserializing)]
|
||||
pub size: i64,
|
||||
}
|
||||
|
||||
pub trait ReplicationWorkerOperation: Any + Send + Sync {
|
||||
fn to_mrf_entry(&self) -> MrfReplicateEntry;
|
||||
fn as_any(&self) -> &dyn Any;
|
||||
fn get_bucket(&self) -> &str;
|
||||
fn get_object(&self) -> &str;
|
||||
fn get_size(&self) -> i64;
|
||||
fn is_delete_marker(&self) -> bool;
|
||||
fn get_op_type(&self) -> ReplicationType;
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ReplicateTargetDecision {
|
||||
pub replicate: bool,
|
||||
pub synchronous: bool,
|
||||
pub arn: String,
|
||||
pub id: String,
|
||||
}
|
||||
|
||||
impl ReplicateTargetDecision {
|
||||
pub fn new(arn: String, replicate: bool, sync: bool) -> Self {
|
||||
Self {
|
||||
replicate,
|
||||
synchronous: sync,
|
||||
arn,
|
||||
id: String::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ReplicateTargetDecision {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
write!(f, "{};{};{};{}", self.replicate, self.synchronous, self.arn, self.id)
|
||||
}
|
||||
}
|
||||
|
||||
/// ReplicateDecision represents replication decision for each target
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicateDecision {
|
||||
pub targets_map: HashMap<String, ReplicateTargetDecision>,
|
||||
}
|
||||
|
||||
impl ReplicateDecision {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
targets_map: HashMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if at least one target qualifies for replication
|
||||
pub fn replicate_any(&self) -> bool {
|
||||
self.targets_map.values().any(|t| t.replicate)
|
||||
}
|
||||
|
||||
/// Returns true if at least one target qualifies for synchronous replication
|
||||
pub fn is_synchronous(&self) -> bool {
|
||||
self.targets_map.values().any(|t| t.synchronous)
|
||||
}
|
||||
|
||||
/// Updates ReplicateDecision with target's replication decision
|
||||
pub fn set(&mut self, target: ReplicateTargetDecision) {
|
||||
self.targets_map.insert(target.arn.clone(), target);
|
||||
}
|
||||
|
||||
/// Returns a stringified representation of internal replication status with all targets marked as `PENDING`
|
||||
pub fn pending_status(&self) -> Option<String> {
|
||||
let mut result = String::new();
|
||||
for target in self.targets_map.values() {
|
||||
if target.replicate {
|
||||
result.push_str(&format!("{}={};", target.arn, ReplicationStatusType::Pending.as_str()));
|
||||
}
|
||||
}
|
||||
if result.is_empty() { None } else { Some(result) }
|
||||
}
|
||||
}
|
||||
|
||||
impl fmt::Display for ReplicateDecision {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
let mut result = String::new();
|
||||
for (key, value) in &self.targets_map {
|
||||
result.push_str(&format!("{key}={value},"));
|
||||
}
|
||||
write!(f, "{}", result.trim_end_matches(','))
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ReplicateDecision {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
// parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a
|
||||
// ReplicateDecision struct
|
||||
pub fn parse_replicate_decision(_bucket: &str, s: &str) -> std::io::Result<ReplicateDecision> {
|
||||
let mut decision = ReplicateDecision::new();
|
||||
|
||||
if s.is_empty() {
|
||||
return Ok(decision);
|
||||
}
|
||||
|
||||
for p in s.split(',') {
|
||||
if p.is_empty() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let slc = p.split('=').collect::<Vec<&str>>();
|
||||
if slc.len() != 2 {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("invalid replicate decision format: {s}"),
|
||||
));
|
||||
}
|
||||
|
||||
let tgt_str = slc[1].trim_matches('"');
|
||||
let tgt = tgt_str.split(';').collect::<Vec<&str>>();
|
||||
if tgt.len() != 4 {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidInput,
|
||||
format!("invalid replicate decision format: {s}"),
|
||||
));
|
||||
}
|
||||
|
||||
let tgt = ReplicateTargetDecision {
|
||||
replicate: tgt[0] == "true",
|
||||
synchronous: tgt[1] == "true",
|
||||
arn: tgt[2].to_string(),
|
||||
id: tgt[3].to_string(),
|
||||
};
|
||||
decision.targets_map.insert(slc[0].to_string(), tgt);
|
||||
}
|
||||
|
||||
Ok(decision)
|
||||
|
||||
// r = ReplicateDecision{
|
||||
// targetsMap: make(map[string]replicateTargetDecision),
|
||||
// }
|
||||
// if len(s) == 0 {
|
||||
// return
|
||||
// }
|
||||
// for _, p := range strings.Split(s, ",") {
|
||||
// if p == "" {
|
||||
// continue
|
||||
// }
|
||||
// slc := strings.Split(p, "=")
|
||||
// if len(slc) != 2 {
|
||||
// return r, errInvalidReplicateDecisionFormat
|
||||
// }
|
||||
// tgtStr := strings.TrimSuffix(strings.TrimPrefix(slc[1], `"`), `"`)
|
||||
// tgt := strings.Split(tgtStr, ";")
|
||||
// if len(tgt) != 4 {
|
||||
// return r, errInvalidReplicateDecisionFormat
|
||||
// }
|
||||
// r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: tgt[0] == "true", Synchronous: tgt[1] == "true", Arn: tgt[2], ID: tgt[3]}
|
||||
// }
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ReplicateObjectInfo {
|
||||
pub name: String,
|
||||
pub size: i64,
|
||||
pub actual_size: i64,
|
||||
pub bucket: String,
|
||||
pub version_id: Option<Uuid>,
|
||||
pub etag: Option<String>,
|
||||
pub mod_time: Option<OffsetDateTime>,
|
||||
pub replication_status: ReplicationStatusType,
|
||||
pub replication_status_internal: Option<String>,
|
||||
pub delete_marker: bool,
|
||||
pub version_purge_status_internal: Option<String>,
|
||||
pub version_purge_status: VersionPurgeStatusType,
|
||||
pub replication_state: Option<ReplicationState>,
|
||||
pub op_type: ReplicationType,
|
||||
pub event_type: String,
|
||||
pub dsc: ReplicateDecision,
|
||||
pub existing_obj_resync: ResyncDecision,
|
||||
pub target_statuses: HashMap<String, ReplicationStatusType>,
|
||||
pub target_purge_statuses: HashMap<String, VersionPurgeStatusType>,
|
||||
pub replication_timestamp: Option<OffsetDateTime>,
|
||||
pub ssec: bool,
|
||||
pub user_tags: String,
|
||||
pub checksum: Option<Bytes>,
|
||||
pub retry_count: u32,
|
||||
}
|
||||
|
||||
impl ReplicationWorkerOperation for ReplicateObjectInfo {
|
||||
fn as_any(&self) -> &dyn Any {
|
||||
self
|
||||
}
|
||||
|
||||
fn to_mrf_entry(&self) -> MrfReplicateEntry {
|
||||
MrfReplicateEntry {
|
||||
bucket: self.bucket.clone(),
|
||||
object: self.name.clone(),
|
||||
version_id: self.version_id,
|
||||
retry_count: self.retry_count as i32,
|
||||
size: self.size,
|
||||
}
|
||||
}
|
||||
|
||||
fn get_bucket(&self) -> &str {
|
||||
&self.bucket
|
||||
}
|
||||
|
||||
fn get_object(&self) -> &str {
|
||||
&self.name
|
||||
}
|
||||
|
||||
fn get_size(&self) -> i64 {
|
||||
self.size
|
||||
}
|
||||
|
||||
fn is_delete_marker(&self) -> bool {
|
||||
self.delete_marker
|
||||
}
|
||||
|
||||
fn get_op_type(&self) -> ReplicationType {
|
||||
self.op_type
|
||||
}
|
||||
}
|
||||
|
||||
lazy_static::lazy_static! {
|
||||
static ref REPL_STATUS_REGEX: Regex = Regex::new(r"([^=].*?)=([^,].*?);").unwrap();
|
||||
}
|
||||
|
||||
impl ReplicateObjectInfo {
|
||||
/// Returns replication status of a target
|
||||
pub fn target_replication_status(&self, arn: &str) -> ReplicationStatusType {
|
||||
let binding = self.replication_status_internal.clone().unwrap_or_default();
|
||||
let captures = REPL_STATUS_REGEX.captures_iter(&binding);
|
||||
for cap in captures {
|
||||
if cap.len() == 3 && &cap[1] == arn {
|
||||
return ReplicationStatusType::from(&cap[2]);
|
||||
}
|
||||
}
|
||||
ReplicationStatusType::default()
|
||||
}
|
||||
|
||||
/// Returns the relevant info needed by MRF
|
||||
pub fn to_mrf_entry(&self) -> MrfReplicateEntry {
|
||||
MrfReplicateEntry {
|
||||
bucket: self.bucket.clone(),
|
||||
object: self.name.clone(),
|
||||
version_id: self.version_id,
|
||||
retry_count: self.retry_count as i32,
|
||||
size: self.size,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// constructs a replication status map from string representation
|
||||
pub fn replication_statuses_map(s: &str) -> HashMap<String, ReplicationStatusType> {
|
||||
let mut targets = HashMap::new();
|
||||
let rep_stat_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
|
||||
for (_, [arn, status]) in rep_stat_matches {
|
||||
if arn.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let status = ReplicationStatusType::from(status);
|
||||
targets.insert(arn.to_string(), status);
|
||||
}
|
||||
targets
|
||||
}
|
||||
|
||||
// constructs a version purge status map from string representation
|
||||
pub fn version_purge_statuses_map(s: &str) -> HashMap<String, VersionPurgeStatusType> {
|
||||
let mut targets = HashMap::new();
|
||||
let purge_status_matches = REPL_STATUS_REGEX.captures_iter(s).map(|c| c.extract());
|
||||
for (_, [arn, status]) in purge_status_matches {
|
||||
if arn.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let status = VersionPurgeStatusType::from(status);
|
||||
targets.insert(arn.to_string(), status);
|
||||
}
|
||||
targets
|
||||
}
|
||||
|
||||
pub fn get_replication_state(rinfos: &ReplicatedInfos, prev_state: &ReplicationState, _vid: Option<String>) -> ReplicationState {
|
||||
let reset_status_map: Vec<(String, String)> = rinfos
|
||||
.targets
|
||||
.iter()
|
||||
.filter(|v| !v.resync_timestamp.is_empty())
|
||||
.map(|t| (target_reset_header(t.arn.as_str()), t.resync_timestamp.clone()))
|
||||
.collect();
|
||||
|
||||
let repl_statuses = rinfos.replication_status_internal();
|
||||
let vpurge_statuses = rinfos.version_purge_status_internal();
|
||||
|
||||
let mut reset_statuses_map = prev_state.reset_statuses_map.clone();
|
||||
for (key, value) in reset_status_map {
|
||||
reset_statuses_map.insert(key, value);
|
||||
}
|
||||
|
||||
ReplicationState {
|
||||
replicate_decision_str: prev_state.replicate_decision_str.clone(),
|
||||
reset_statuses_map,
|
||||
replica_timestamp: prev_state.replica_timestamp,
|
||||
replica_status: prev_state.replica_status.clone(),
|
||||
targets: replication_statuses_map(&repl_statuses.clone().unwrap_or_default()),
|
||||
replication_status_internal: repl_statuses,
|
||||
replication_timestamp: rinfos.replication_timestamp,
|
||||
purge_targets: version_purge_statuses_map(&vpurge_statuses.clone().unwrap_or_default()),
|
||||
version_purge_status_internal: vpurge_statuses,
|
||||
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn target_reset_header(arn: &str) -> String {
|
||||
format!("{RESERVED_METADATA_PREFIX_LOWER}{REPLICATION_RESET}-{arn}")
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default)]
|
||||
pub struct ResyncTargetDecision {
|
||||
pub replicate: bool,
|
||||
pub reset_id: String,
|
||||
pub reset_before_date: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
/// ResyncDecision is a struct representing a map with target's individual resync decisions
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ResyncDecision {
|
||||
pub targets: HashMap<String, ResyncTargetDecision>,
|
||||
}
|
||||
|
||||
impl ResyncDecision {
|
||||
pub fn new() -> Self {
|
||||
Self { targets: HashMap::new() }
|
||||
}
|
||||
|
||||
/// Returns true if no targets with resync decision present
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.targets.is_empty()
|
||||
}
|
||||
|
||||
pub fn must_resync(&self) -> bool {
|
||||
self.targets.values().any(|v| v.replicate)
|
||||
}
|
||||
|
||||
pub fn must_resync_target(&self, tgt_arn: &str) -> bool {
|
||||
self.targets.get(tgt_arn).map(|v| v.replicate).unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ResyncDecision {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,7 +34,6 @@ time = { workspace = true, features = ["serde-human-readable"] }
|
||||
serde = { workspace = true, features = ["derive", "rc"] }
|
||||
rustfs-ecstore = { workspace = true }
|
||||
rustfs-policy.workspace = true
|
||||
rustfs-config.workspace = true
|
||||
serde_json.workspace = true
|
||||
async-trait.workspace = true
|
||||
thiserror.workspace = true
|
||||
|
||||
@@ -33,6 +33,7 @@ use rustfs_policy::{
|
||||
EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, Policy, PolicyDoc, default::DEFAULT_POLICIES, iam_policy_claim_name_sa,
|
||||
},
|
||||
};
|
||||
use rustfs_utils::crypto::base64_encode;
|
||||
use rustfs_utils::path::path_join_buf;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
@@ -554,10 +555,7 @@ where
|
||||
return Err(Error::PolicyTooLarge);
|
||||
}
|
||||
|
||||
m.insert(
|
||||
SESSION_POLICY_NAME.to_owned(),
|
||||
Value::String(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&policy_buf)),
|
||||
);
|
||||
m.insert(SESSION_POLICY_NAME.to_owned(), Value::String(base64_encode(&policy_buf)));
|
||||
m.insert(iam_policy_claim_name_sa(), Value::String(EMBEDDED_POLICY_TYPE.to_owned()));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -35,16 +35,14 @@ use rustfs_policy::auth::{
|
||||
is_access_key_valid, is_secret_key_valid,
|
||||
};
|
||||
use rustfs_policy::policy::Args;
|
||||
use rustfs_policy::policy::opa;
|
||||
use rustfs_policy::policy::{EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, Policy, PolicyDoc, iam_policy_claim_name_sa};
|
||||
use rustfs_utils::crypto::{base64_decode, base64_encode};
|
||||
use serde_json::Value;
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
use std::sync::OnceLock;
|
||||
use time::OffsetDateTime;
|
||||
use tokio::sync::RwLock;
|
||||
use tracing::{error, info, warn};
|
||||
use tracing::warn;
|
||||
|
||||
pub const MAX_SVCSESSION_POLICY_SIZE: usize = 4096;
|
||||
|
||||
@@ -55,12 +53,6 @@ pub const POLICYNAME: &str = "policy";
|
||||
pub const SESSION_POLICY_NAME: &str = "sessionPolicy";
|
||||
pub const SESSION_POLICY_NAME_EXTRACTED: &str = "sessionPolicy-extracted";
|
||||
|
||||
static POLICY_PLUGIN_CLIENT: OnceLock<Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>>> = OnceLock::new();
|
||||
|
||||
fn get_policy_plugin_client() -> Arc<RwLock<Option<rustfs_policy::policy::opa::AuthZPlugin>>> {
|
||||
POLICY_PLUGIN_CLIENT.get_or_init(|| Arc::new(RwLock::new(None))).clone()
|
||||
}
|
||||
|
||||
pub struct IamSys<T> {
|
||||
store: Arc<IamCache<T>>,
|
||||
roles_map: HashMap<ARN, String>,
|
||||
@@ -68,20 +60,6 @@ pub struct IamSys<T> {
|
||||
|
||||
impl<T: Store> IamSys<T> {
|
||||
pub fn new(store: Arc<IamCache<T>>) -> Self {
|
||||
tokio::spawn(async move {
|
||||
match opa::lookup_config().await {
|
||||
Ok(conf) => {
|
||||
if conf.enable() {
|
||||
Self::set_policy_plugin_client(opa::AuthZPlugin::new(conf)).await;
|
||||
info!("OPA plugin enabled");
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Error loading OPA configuration err:{}", e);
|
||||
}
|
||||
};
|
||||
});
|
||||
|
||||
Self {
|
||||
store,
|
||||
roles_map: HashMap::new(),
|
||||
@@ -91,18 +69,6 @@ impl<T: Store> IamSys<T> {
|
||||
self.store.api.has_watcher()
|
||||
}
|
||||
|
||||
pub async fn set_policy_plugin_client(client: rustfs_policy::policy::opa::AuthZPlugin) {
|
||||
let policy_plugin_client = get_policy_plugin_client();
|
||||
let mut guard = policy_plugin_client.write().await;
|
||||
*guard = Some(client);
|
||||
}
|
||||
|
||||
pub async fn get_policy_plugin_client() -> Option<rustfs_policy::policy::opa::AuthZPlugin> {
|
||||
let policy_plugin_client = get_policy_plugin_client();
|
||||
let guard = policy_plugin_client.read().await;
|
||||
guard.clone()
|
||||
}
|
||||
|
||||
pub async fn load_group(&self, name: &str) -> Result<()> {
|
||||
self.store.group_notification_handler(name).await
|
||||
}
|
||||
@@ -362,10 +328,7 @@ impl<T: Store> IamSys<T> {
|
||||
m.insert("parent".to_owned(), Value::String(parent_user.to_owned()));
|
||||
|
||||
if !policy_buf.is_empty() {
|
||||
m.insert(
|
||||
SESSION_POLICY_NAME.to_owned(),
|
||||
Value::String(base64_simd::URL_SAFE_NO_PAD.encode_to_string(&policy_buf)),
|
||||
);
|
||||
m.insert(SESSION_POLICY_NAME.to_owned(), Value::String(base64_encode(&policy_buf)));
|
||||
m.insert(iam_policy_claim_name_sa(), Value::String(EMBEDDED_POLICY_TYPE.to_owned()));
|
||||
} else {
|
||||
m.insert(iam_policy_claim_name_sa(), Value::String(INHERITED_POLICY_TYPE.to_owned()));
|
||||
@@ -458,9 +421,7 @@ impl<T: Store> IamSys<T> {
|
||||
let op_sp = claims.get(SESSION_POLICY_NAME);
|
||||
if let (Some(pt), Some(sp)) = (op_pt, op_sp) {
|
||||
if pt == EMBEDDED_POLICY_TYPE {
|
||||
let policy = serde_json::from_slice(
|
||||
&base64_simd::URL_SAFE_NO_PAD.decode_to_vec(sp.as_str().unwrap_or_default().as_bytes())?,
|
||||
)?;
|
||||
let policy = serde_json::from_slice(&base64_decode(sp.as_str().unwrap_or_default().as_bytes())?)?;
|
||||
return Ok((sa, Some(policy)));
|
||||
}
|
||||
}
|
||||
@@ -519,9 +480,7 @@ impl<T: Store> IamSys<T> {
|
||||
let op_sp = claims.get(SESSION_POLICY_NAME);
|
||||
if let (Some(pt), Some(sp)) = (op_pt, op_sp) {
|
||||
if pt == EMBEDDED_POLICY_TYPE {
|
||||
let policy = serde_json::from_slice(
|
||||
&base64_simd::URL_SAFE_NO_PAD.decode_to_vec(sp.as_str().unwrap_or_default().as_bytes())?,
|
||||
)?;
|
||||
let policy = serde_json::from_slice(&base64_decode(sp.as_str().unwrap_or_default().as_bytes())?)?;
|
||||
return Ok((sa, Some(policy)));
|
||||
}
|
||||
}
|
||||
@@ -534,7 +493,7 @@ impl<T: Store> IamSys<T> {
|
||||
return Err(IamError::NoSuchServiceAccount(access_key.to_string()));
|
||||
};
|
||||
|
||||
if !u.credentials.is_service_account() {
|
||||
if u.credentials.is_service_account() {
|
||||
return Err(IamError::NoSuchServiceAccount(access_key.to_string()));
|
||||
}
|
||||
|
||||
@@ -807,11 +766,6 @@ impl<T: Store> IamSys<T> {
|
||||
return true;
|
||||
}
|
||||
|
||||
let opa_enable = Self::get_policy_plugin_client().await;
|
||||
if let Some(opa_enable) = opa_enable {
|
||||
return opa_enable.is_allowed(args).await;
|
||||
}
|
||||
|
||||
let Ok((is_temp, parent_user)) = self.is_temp_user(args.account).await else { return false };
|
||||
|
||||
if is_temp {
|
||||
@@ -912,9 +866,7 @@ pub fn get_claims_from_token_with_secret(token: &str, secret: &str) -> Result<Ha
|
||||
|
||||
if let Some(session_policy) = ms.claims.get(SESSION_POLICY_NAME) {
|
||||
let policy_str = session_policy.as_str().unwrap_or_default();
|
||||
let policy = base64_simd::URL_SAFE_NO_PAD
|
||||
.decode_to_vec(policy_str.as_bytes())
|
||||
.map_err(|e| Error::other(format!("base64 decode err {e}")))?;
|
||||
let policy = base64_decode(policy_str.as_bytes()).map_err(|e| Error::other(format!("base64 decode err {e}")))?;
|
||||
ms.claims.insert(
|
||||
SESSION_POLICY_NAME_EXTRACTED.to_string(),
|
||||
Value::String(String::from_utf8(policy).map_err(|e| Error::other(format!("utf8 decode err {e}")))?),
|
||||
|
||||
@@ -635,7 +635,7 @@ impl KmsBackend for LocalKmsBackend {
|
||||
}
|
||||
|
||||
async fn encrypt(&self, request: EncryptRequest) -> Result<EncryptResponse> {
|
||||
let encrypt_request = EncryptRequest {
|
||||
let encrypt_request = crate::types::EncryptRequest {
|
||||
key_id: request.key_id.clone(),
|
||||
plaintext: request.plaintext,
|
||||
encryption_context: request.encryption_context,
|
||||
@@ -719,14 +719,14 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.client
|
||||
.load_master_key(key_id)
|
||||
.await
|
||||
.map_err(|_| KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
.map_err(|_| crate::error::KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
|
||||
let (deletion_date_str, deletion_date_dt) = if request.force_immediate.unwrap_or(false) {
|
||||
// For immediate deletion, actually delete the key from filesystem
|
||||
let key_path = self.client.master_key_path(key_id);
|
||||
tokio::fs::remove_file(&key_path)
|
||||
.await
|
||||
.map_err(|e| KmsError::internal_error(format!("Failed to delete key file: {e}")))?;
|
||||
.map_err(|e| crate::error::KmsError::internal_error(format!("Failed to delete key file: {e}")))?;
|
||||
|
||||
// Remove from cache
|
||||
let mut cache = self.client.key_cache.write().await;
|
||||
@@ -756,7 +756,9 @@ impl KmsBackend for LocalKmsBackend {
|
||||
// Schedule for deletion (default 30 days)
|
||||
let days = request.pending_window_in_days.unwrap_or(30);
|
||||
if !(7..=30).contains(&days) {
|
||||
return Err(KmsError::invalid_parameter("pending_window_in_days must be between 7 and 30".to_string()));
|
||||
return Err(crate::error::KmsError::invalid_parameter(
|
||||
"pending_window_in_days must be between 7 and 30".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
let deletion_date = chrono::Utc::now() + chrono::Duration::days(days as i64);
|
||||
@@ -770,16 +772,16 @@ impl KmsBackend for LocalKmsBackend {
|
||||
let key_path = self.client.master_key_path(key_id);
|
||||
let content = tokio::fs::read(&key_path)
|
||||
.await
|
||||
.map_err(|e| KmsError::internal_error(format!("Failed to read key file: {e}")))?;
|
||||
let stored_key: StoredMasterKey =
|
||||
serde_json::from_slice(&content).map_err(|e| KmsError::internal_error(format!("Failed to parse stored key: {e}")))?;
|
||||
.map_err(|e| crate::error::KmsError::internal_error(format!("Failed to read key file: {e}")))?;
|
||||
let stored_key: crate::backends::local::StoredMasterKey = serde_json::from_slice(&content)
|
||||
.map_err(|e| crate::error::KmsError::internal_error(format!("Failed to parse stored key: {e}")))?;
|
||||
|
||||
// Decrypt the existing key material to preserve it
|
||||
let existing_key_material = if let Some(ref cipher) = self.client.master_cipher {
|
||||
let nonce = Nonce::from_slice(&stored_key.nonce);
|
||||
let nonce = aes_gcm::Nonce::from_slice(&stored_key.nonce);
|
||||
cipher
|
||||
.decrypt(nonce, stored_key.encrypted_key_material.as_ref())
|
||||
.map_err(|e| KmsError::cryptographic_error("decrypt", e.to_string()))?
|
||||
.map_err(|e| crate::error::KmsError::cryptographic_error("decrypt", e.to_string()))?
|
||||
} else {
|
||||
stored_key.encrypted_key_material
|
||||
};
|
||||
@@ -818,10 +820,10 @@ impl KmsBackend for LocalKmsBackend {
|
||||
.client
|
||||
.load_master_key(key_id)
|
||||
.await
|
||||
.map_err(|_| KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
.map_err(|_| crate::error::KmsError::key_not_found(format!("Key {key_id} not found")))?;
|
||||
|
||||
if master_key.status != KeyStatus::PendingDeletion {
|
||||
return Err(KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion")));
|
||||
return Err(crate::error::KmsError::invalid_key_state(format!("Key {key_id} is not pending deletion")));
|
||||
}
|
||||
|
||||
// Cancel the deletion by resetting the state
|
||||
|
||||
@@ -68,7 +68,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
key: WEBHOOK_QUEUE_DIR.to_string(),
|
||||
value: current_root
|
||||
.clone()
|
||||
.join("../../deploy/logs/notify")
|
||||
.join("../../deploy/logs/notify/webhook")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
@@ -120,7 +120,11 @@ async fn main() -> Result<(), NotificationError> {
|
||||
},
|
||||
KV {
|
||||
key: MQTT_QUEUE_DIR.to_string(),
|
||||
value: current_root.join("../../deploy/logs/notify").to_str().unwrap().to_string(),
|
||||
value: current_root
|
||||
.join("../../deploy/logs/notify/mqtt")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
@@ -133,7 +137,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
let mqtt_kvs = KVS(mqtt_kvs_vec);
|
||||
let mut mqtt_targets = std::collections::HashMap::new();
|
||||
mqtt_targets.insert(DEFAULT_TARGET.to_string(), mqtt_kvs);
|
||||
// config.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_targets);
|
||||
config.0.insert(NOTIFY_MQTT_SUB_SYS.to_string(), mqtt_targets);
|
||||
|
||||
// Load the configuration and initialize the system
|
||||
*system.config.write().await = config;
|
||||
|
||||
@@ -28,7 +28,6 @@ use rustfs_targets::EventName;
|
||||
use rustfs_targets::arn::TargetID;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
use tokio::time::sleep;
|
||||
use tracing::info;
|
||||
|
||||
#[tokio::main]
|
||||
@@ -69,7 +68,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
key: WEBHOOK_QUEUE_DIR.to_string(),
|
||||
value: current_root
|
||||
.clone()
|
||||
.join("../../deploy/logs/notify")
|
||||
.join("../../deploy/logs/notify/webhook")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
@@ -92,7 +91,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
system.init().await?;
|
||||
info!("✅ System initialized with Webhook target.");
|
||||
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
|
||||
// --- Dynamically update system configuration: Add an MQTT Target ---
|
||||
info!("\n---> Dynamically adding MQTT target...");
|
||||
@@ -130,7 +129,11 @@ async fn main() -> Result<(), NotificationError> {
|
||||
},
|
||||
KV {
|
||||
key: MQTT_QUEUE_DIR.to_string(),
|
||||
value: current_root.join("../../deploy/logs/notify").to_str().unwrap().to_string(),
|
||||
value: current_root
|
||||
.join("../../deploy/logs/notify/mqtt")
|
||||
.to_str()
|
||||
.unwrap()
|
||||
.to_string(),
|
||||
hidden_if_empty: false,
|
||||
},
|
||||
KV {
|
||||
@@ -149,7 +152,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
.await?;
|
||||
info!("✅ MQTT target added and system reloaded.");
|
||||
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
tokio::time::sleep(Duration::from_secs(1)).await;
|
||||
|
||||
// --- Loading and managing Bucket configurations ---
|
||||
info!("\n---> Loading bucket notification config...");
|
||||
@@ -173,7 +176,7 @@ async fn main() -> Result<(), NotificationError> {
|
||||
system.send_event(event).await;
|
||||
info!("✅ Event sent. Both Webhook and MQTT targets should receive it.");
|
||||
|
||||
sleep(Duration::from_secs(2)).await;
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
|
||||
// --- Dynamically remove configuration ---
|
||||
info!("\n---> Dynamically removing Webhook target...");
|
||||
@@ -185,6 +188,5 @@ async fn main() -> Result<(), NotificationError> {
|
||||
info!("✅ Bucket 'my-bucket' config removed.");
|
||||
|
||||
info!("\nDemo completed successfully");
|
||||
sleep(Duration::from_secs(1)).await;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -12,20 +12,19 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use axum::routing::get;
|
||||
use axum::{
|
||||
Router,
|
||||
extract::Json,
|
||||
extract::Query,
|
||||
http::{HeaderMap, Response, StatusCode},
|
||||
routing::{get, post},
|
||||
routing::post,
|
||||
};
|
||||
use rustfs_utils::parse_and_resolve_address;
|
||||
use serde::Deserialize;
|
||||
use serde_json::Value;
|
||||
use std::net::SocketAddr;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::time::{SystemTime, UNIX_EPOCH};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
use axum::extract::Query;
|
||||
use serde::Deserialize;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct ResetParams {
|
||||
@@ -33,6 +32,9 @@ struct ResetParams {
|
||||
}
|
||||
|
||||
// Define a global variable and count the number of data received
|
||||
use rustfs_utils::parse_and_resolve_address;
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use tokio::net::TcpListener;
|
||||
|
||||
static WEBHOOK_COUNT: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
|
||||
@@ -296,8 +296,8 @@ impl NotificationSystem {
|
||||
info!("Removing config for target {} of type {}", target_name, target_type);
|
||||
self.update_config_and_reload(|config| {
|
||||
let mut changed = false;
|
||||
if let Some(targets) = config.0.get_mut(&target_type.to_lowercase()) {
|
||||
if targets.remove(&target_name.to_lowercase()).is_some() {
|
||||
if let Some(targets) = config.0.get_mut(target_type) {
|
||||
if targets.remove(target_name).is_some() {
|
||||
changed = true;
|
||||
}
|
||||
if targets.is_empty() {
|
||||
@@ -307,7 +307,6 @@ impl NotificationSystem {
|
||||
if !changed {
|
||||
info!("Target {} of type {} not found, no changes made.", target_name, target_type);
|
||||
}
|
||||
debug!("Config after remove: {:?}", config);
|
||||
changed
|
||||
})
|
||||
.await
|
||||
|
||||
@@ -16,9 +16,12 @@ use crate::Event;
|
||||
use crate::factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory};
|
||||
use futures::stream::{FuturesUnordered, StreamExt};
|
||||
use hashbrown::{HashMap, HashSet};
|
||||
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX, notify::NOTIFY_ROUTE_PREFIX};
|
||||
use rustfs_config::notify::NOTIFY_ROUTE_PREFIX;
|
||||
use rustfs_config::{DEFAULT_DELIMITER, ENABLE_KEY, ENV_PREFIX};
|
||||
use rustfs_ecstore::config::{Config, KVS};
|
||||
use rustfs_targets::{Target, TargetError, target::ChannelTargetType};
|
||||
use rustfs_targets::Target;
|
||||
use rustfs_targets::TargetError;
|
||||
use rustfs_targets::target::ChannelTargetType;
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
/// Registry for managing target factories
|
||||
@@ -87,9 +90,7 @@ impl TargetRegistry {
|
||||
let all_env: Vec<(String, String)> = std::env::vars().filter(|(key, _)| key.starts_with(ENV_PREFIX)).collect();
|
||||
// A collection of asynchronous tasks for concurrently executing target creation
|
||||
let mut tasks = FuturesUnordered::new();
|
||||
// let final_config = config.clone(); // Clone a configuration for aggregating the final result
|
||||
// Record the defaults for each segment so that the segment can eventually be rebuilt
|
||||
let mut section_defaults: HashMap<String, KVS> = HashMap::new();
|
||||
let mut final_config = config.clone(); // Clone a configuration for aggregating the final result
|
||||
// 1. Traverse all registered plants and process them by target type
|
||||
for (target_type, factory) in &self.factories {
|
||||
tracing::Span::current().record("target_type", target_type.as_str());
|
||||
@@ -97,15 +98,12 @@ impl TargetRegistry {
|
||||
|
||||
// 2. Prepare the configuration source
|
||||
// 2.1. Get the configuration segment in the file, e.g. 'notify_webhook'
|
||||
let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}").to_lowercase();
|
||||
let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}");
|
||||
let file_configs = config.0.get(§ion_name).cloned().unwrap_or_default();
|
||||
// 2.2. Get the default configuration for that type
|
||||
let default_cfg = file_configs.get(DEFAULT_DELIMITER).cloned().unwrap_or_default();
|
||||
debug!(?default_cfg, "Get the default configuration");
|
||||
|
||||
// Save defaults for eventual write back
|
||||
section_defaults.insert(section_name.clone(), default_cfg.clone());
|
||||
|
||||
// *** Optimization point 1: Get all legitimate fields of the current target type ***
|
||||
let valid_fields = factory.get_valid_fields();
|
||||
debug!(?valid_fields, "Get the legitimate configuration fields");
|
||||
@@ -113,9 +111,7 @@ impl TargetRegistry {
|
||||
// 3. Resolve instance IDs and configuration overrides from environment variables
|
||||
let mut instance_ids_from_env = HashSet::new();
|
||||
// 3.1. Instance discovery: Based on the '..._ENABLE_INSTANCEID' format
|
||||
let enable_prefix =
|
||||
format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}{ENABLE_KEY}{DEFAULT_DELIMITER}")
|
||||
.to_uppercase();
|
||||
let enable_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}_{ENABLE_KEY}_").to_uppercase();
|
||||
for (key, value) in &all_env {
|
||||
if value.eq_ignore_ascii_case(rustfs_config::EnableState::One.as_str())
|
||||
|| value.eq_ignore_ascii_case(rustfs_config::EnableState::On.as_str())
|
||||
@@ -132,14 +128,14 @@ impl TargetRegistry {
|
||||
|
||||
// 3.2. Parse all relevant environment variable configurations
|
||||
// 3.2.1. Build environment variable prefixes such as 'RUSTFS_NOTIFY_WEBHOOK_'
|
||||
let env_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}{DEFAULT_DELIMITER}").to_uppercase();
|
||||
let env_prefix = format!("{ENV_PREFIX}{NOTIFY_ROUTE_PREFIX}{target_type}_").to_uppercase();
|
||||
// 3.2.2. 'env_overrides' is used to store configurations parsed from environment variables in the format: {instance id -> {field -> value}}
|
||||
let mut env_overrides: HashMap<String, HashMap<String, String>> = HashMap::new();
|
||||
for (key, value) in &all_env {
|
||||
if let Some(rest) = key.strip_prefix(&env_prefix) {
|
||||
// Use rsplitn to split from the right side to properly extract the INSTANCE_ID at the end
|
||||
// Format: <FIELD_NAME>_<INSTANCE_ID> or <FIELD_NAME>
|
||||
let mut parts = rest.rsplitn(2, DEFAULT_DELIMITER);
|
||||
let mut parts = rest.rsplitn(2, '_');
|
||||
|
||||
// The first part from the right is INSTANCE_ID
|
||||
let instance_id_part = parts.next().unwrap_or(DEFAULT_DELIMITER);
|
||||
@@ -228,7 +224,7 @@ impl TargetRegistry {
|
||||
} else {
|
||||
info!(instance_id = %id, "Skip the disabled target and will be removed from the final configuration");
|
||||
// Remove disabled target from final configuration
|
||||
// final_config.0.entry(section_name.clone()).or_default().remove(&id);
|
||||
final_config.0.entry(section_name.clone()).or_default().remove(&id);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -250,50 +246,15 @@ impl TargetRegistry {
|
||||
}
|
||||
|
||||
// 7. Aggregate new configuration and write back to system configuration
|
||||
if !successful_configs.is_empty() || !section_defaults.is_empty() {
|
||||
if !successful_configs.is_empty() {
|
||||
info!(
|
||||
"Prepare to update {} successfully created target configurations to the system configuration...",
|
||||
successful_configs.len()
|
||||
);
|
||||
|
||||
let mut successes_by_section: HashMap<String, HashMap<String, KVS>> = HashMap::new();
|
||||
|
||||
let mut new_config = config.clone();
|
||||
for (target_type, id, kvs) in successful_configs {
|
||||
let section_name = format!("{NOTIFY_ROUTE_PREFIX}{target_type}").to_lowercase();
|
||||
successes_by_section
|
||||
.entry(section_name)
|
||||
.or_default()
|
||||
.insert(id.to_lowercase(), (*kvs).clone());
|
||||
}
|
||||
|
||||
let mut new_config = config.clone();
|
||||
// Collection of segments that need to be processed: Collect all segments where default items exist or where successful instances exist
|
||||
let mut sections: HashSet<String> = HashSet::new();
|
||||
sections.extend(section_defaults.keys().cloned());
|
||||
sections.extend(successes_by_section.keys().cloned());
|
||||
|
||||
for section in sections {
|
||||
let mut section_map: std::collections::HashMap<String, KVS> = std::collections::HashMap::new();
|
||||
// Add default item
|
||||
if let Some(default_kvs) = section_defaults.get(§ion) {
|
||||
if !default_kvs.is_empty() {
|
||||
section_map.insert(DEFAULT_DELIMITER.to_string(), default_kvs.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Add successful instance item
|
||||
if let Some(instances) = successes_by_section.get(§ion) {
|
||||
for (id, kvs) in instances {
|
||||
section_map.insert(id.clone(), kvs.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Empty breaks are removed and non-empty breaks are replaced entirely.
|
||||
if section_map.is_empty() {
|
||||
new_config.0.remove(§ion);
|
||||
} else {
|
||||
new_config.0.insert(section, section_map);
|
||||
}
|
||||
new_config.0.entry(section_name).or_default().insert(id, (*kvs).clone());
|
||||
}
|
||||
|
||||
let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else {
|
||||
|
||||
@@ -29,12 +29,17 @@ documentation = "https://docs.rs/rustfs-obs/latest/rustfs_obs/"
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
default = ["file"]
|
||||
file = []
|
||||
gpu = ["dep:nvml-wrapper"]
|
||||
webhook = ["dep:reqwest"]
|
||||
kafka = ["dep:rdkafka"]
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["constants", "observability"] }
|
||||
rustfs-utils = { workspace = true, features = ["ip", "path"] }
|
||||
async-trait = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
flexi_logger = { workspace = true }
|
||||
nu-ansi-term = { workspace = true }
|
||||
nvml-wrapper = { workspace = true, optional = true }
|
||||
@@ -52,9 +57,24 @@ tracing-error = { workspace = true }
|
||||
tracing-opentelemetry = { workspace = true }
|
||||
tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
|
||||
reqwest = { workspace = true, optional = true }
|
||||
serde_json = { workspace = true }
|
||||
sysinfo = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
|
||||
|
||||
# Only enable kafka features and related dependencies on Linux
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
rdkafka = { workspace = true, features = ["tokio"], optional = true }
|
||||
|
||||
|
||||
[dev-dependencies]
|
||||
chrono = { workspace = true }
|
||||
opentelemetry = { workspace = true }
|
||||
opentelemetry_sdk = { workspace = true, features = ["rt-tokio"] }
|
||||
opentelemetry-stdout = { workspace = true }
|
||||
opentelemetry-otlp = { workspace = true, features = ["grpc-tonic"] }
|
||||
opentelemetry-semantic-conventions = { workspace = true, features = ["semconv_experimental"] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
tracing = { workspace = true, features = ["std", "attributes"] }
|
||||
tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt"] }
|
||||
@@ -21,4 +21,29 @@ service_name = "rustfs"
|
||||
service_version = "0.1.0"
|
||||
environments = "develop"
|
||||
logger_level = "debug"
|
||||
local_logging_enabled = true # Default is false if not specified
|
||||
local_logging_enabled = true # Default is false if not specified
|
||||
|
||||
|
||||
#[[sinks]]
|
||||
#type = "Kafka"
|
||||
#bootstrap_servers = "localhost:9092"
|
||||
#topic = "logs"
|
||||
#batch_size = 100 # Default is 100 if not specified
|
||||
#batch_timeout_ms = 100 # Default is 1000ms if not specified
|
||||
#
|
||||
#[[sinks]]
|
||||
#type = "Webhook"
|
||||
#endpoint = "http://localhost:8080/webhook"
|
||||
#auth_token = ""
|
||||
#batch_size = 100 # Default is 3 if not specified
|
||||
#batch_timeout_ms = 100 # Default is 100ms if not specified
|
||||
|
||||
[[sinks]]
|
||||
type = "File"
|
||||
path = "deploy/logs/rustfs.log"
|
||||
buffer_size = 102 # Default is 8192 bytes if not specified
|
||||
flush_interval_ms = 1000
|
||||
flush_threshold = 100
|
||||
|
||||
[logger]
|
||||
queue_capacity = 10000
|
||||
@@ -13,25 +13,33 @@
|
||||
// limitations under the License.
|
||||
|
||||
use opentelemetry::global;
|
||||
use rustfs_obs::{SystemObserver, init_obs};
|
||||
use rustfs_obs::{BaseLogEntry, ServerLogEntry, SystemObserver, get_logger, init_obs, log_info};
|
||||
use std::collections::HashMap;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use tracing::{Level, error, info, instrument};
|
||||
use tracing::{error, info, instrument};
|
||||
use tracing_core::Level;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() {
|
||||
let obs_conf = Some("http://localhost:4317".to_string());
|
||||
let _guard = init_obs(obs_conf).await;
|
||||
let obs_conf = Some("crates/obs/examples/config.toml".to_string());
|
||||
let (_logger, _guard) = init_obs(obs_conf).await;
|
||||
let span = tracing::span!(Level::INFO, "main");
|
||||
let _enter = span.enter();
|
||||
info!("Program starts");
|
||||
// Simulate the operation
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
run("service-demo".to_string()).await;
|
||||
run(
|
||||
"service-demo".to_string(),
|
||||
"object-demo".to_string(),
|
||||
"user-demo".to_string(),
|
||||
"service-demo".to_string(),
|
||||
)
|
||||
.await;
|
||||
info!("Program ends");
|
||||
}
|
||||
|
||||
#[instrument(fields(bucket, object, user))]
|
||||
async fn run(service_name: String) {
|
||||
async fn run(bucket: String, object: String, user: String, service_name: String) {
|
||||
let start_time = SystemTime::now();
|
||||
info!("Log module initialization is completed service_name: {:?}", service_name);
|
||||
|
||||
@@ -48,6 +56,21 @@ async fn run(service_name: String) {
|
||||
Err(e) => error!("Failed to initialize process observer: {:?}", e),
|
||||
}
|
||||
|
||||
let base_entry = BaseLogEntry::new()
|
||||
.message(Some("run logger api_handler info".to_string()))
|
||||
.request_id(Some("request_id".to_string()))
|
||||
.timestamp(chrono::DateTime::from(start_time))
|
||||
.tags(Some(HashMap::default()));
|
||||
|
||||
let server_entry = ServerLogEntry::new(Level::INFO, "api_handler".to_string())
|
||||
.with_base(base_entry)
|
||||
.user_id(Some(user.clone()))
|
||||
.add_field("operation".to_string(), "login".to_string())
|
||||
.add_field("bucket".to_string(), bucket.clone())
|
||||
.add_field("object".to_string(), object.clone());
|
||||
|
||||
let result = get_logger().lock().await.log_server_entry(server_entry).await;
|
||||
info!("Logging is completed {:?}", result);
|
||||
put_object("bucket".to_string(), "object".to_string(), "user".to_string()).await;
|
||||
info!("Logging is completed");
|
||||
tokio::time::sleep(Duration::from_secs(2)).await;
|
||||
@@ -74,6 +97,8 @@ async fn put_object(bucket: String, object: String, user: String) {
|
||||
start_time.elapsed().unwrap().as_secs_f64()
|
||||
);
|
||||
|
||||
let result = log_info("put_object logger info", "put_object").await;
|
||||
info!("put_object is completed {:?}", result);
|
||||
// Simulate the operation
|
||||
tokio::time::sleep(Duration::from_millis(100)).await;
|
||||
|
||||
|
||||
@@ -13,9 +13,16 @@
|
||||
// limitations under the License.
|
||||
|
||||
use rustfs_config::observability::{
|
||||
ENV_OBS_ENDPOINT, ENV_OBS_ENVIRONMENT, ENV_OBS_LOCAL_LOGGING_ENABLED, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_FILENAME,
|
||||
ENV_OBS_LOG_KEEP_FILES, ENV_OBS_LOG_ROTATION_SIZE_MB, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOGGER_LEVEL,
|
||||
ENV_OBS_METER_INTERVAL, ENV_OBS_SAMPLE_RATIO, ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_USE_STDOUT,
|
||||
DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY, DEFAULT_SINKS_FILE_BUFFER_SIZE, DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS,
|
||||
DEFAULT_SINKS_FILE_FLUSH_THRESHOLD, DEFAULT_SINKS_KAFKA_BATCH_SIZE, DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS,
|
||||
DEFAULT_SINKS_KAFKA_BROKERS, DEFAULT_SINKS_KAFKA_TOPIC, DEFAULT_SINKS_WEBHOOK_AUTH_TOKEN, DEFAULT_SINKS_WEBHOOK_ENDPOINT,
|
||||
DEFAULT_SINKS_WEBHOOK_MAX_RETRIES, DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS, ENV_AUDIT_LOGGER_QUEUE_CAPACITY, ENV_OBS_ENDPOINT,
|
||||
ENV_OBS_ENVIRONMENT, ENV_OBS_LOCAL_LOGGING_ENABLED, ENV_OBS_LOG_DIRECTORY, ENV_OBS_LOG_FILENAME, ENV_OBS_LOG_KEEP_FILES,
|
||||
ENV_OBS_LOG_ROTATION_SIZE_MB, ENV_OBS_LOG_ROTATION_TIME, ENV_OBS_LOGGER_LEVEL, ENV_OBS_METER_INTERVAL, ENV_OBS_SAMPLE_RATIO,
|
||||
ENV_OBS_SERVICE_NAME, ENV_OBS_SERVICE_VERSION, ENV_OBS_USE_STDOUT, ENV_SINKS_FILE_BUFFER_SIZE,
|
||||
ENV_SINKS_FILE_FLUSH_INTERVAL_MS, ENV_SINKS_FILE_FLUSH_THRESHOLD, ENV_SINKS_FILE_PATH, ENV_SINKS_KAFKA_BATCH_SIZE,
|
||||
ENV_SINKS_KAFKA_BATCH_TIMEOUT_MS, ENV_SINKS_KAFKA_BROKERS, ENV_SINKS_KAFKA_TOPIC, ENV_SINKS_WEBHOOK_AUTH_TOKEN,
|
||||
ENV_SINKS_WEBHOOK_ENDPOINT, ENV_SINKS_WEBHOOK_MAX_RETRIES, ENV_SINKS_WEBHOOK_RETRY_DELAY_MS,
|
||||
};
|
||||
use rustfs_config::{
|
||||
APP_NAME, DEFAULT_LOG_KEEP_FILES, DEFAULT_LOG_LEVEL, DEFAULT_LOG_ROTATION_SIZE_MB, DEFAULT_LOG_ROTATION_TIME,
|
||||
@@ -138,10 +145,167 @@ impl Default for OtelConfig {
|
||||
}
|
||||
}
|
||||
|
||||
/// Kafka Sink Configuration - Add batch parameters
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct KafkaSinkConfig {
|
||||
pub brokers: String,
|
||||
pub topic: String,
|
||||
pub batch_size: Option<usize>, // Batch size, default 100
|
||||
pub batch_timeout_ms: Option<u64>, // Batch timeout time, default 1000ms
|
||||
}
|
||||
|
||||
impl KafkaSinkConfig {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for KafkaSinkConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
brokers: env::var(ENV_SINKS_KAFKA_BROKERS)
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_SINKS_KAFKA_BROKERS.to_string()),
|
||||
topic: env::var(ENV_SINKS_KAFKA_TOPIC)
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_SINKS_KAFKA_TOPIC.to_string()),
|
||||
batch_size: env::var(ENV_SINKS_KAFKA_BATCH_SIZE)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_KAFKA_BATCH_SIZE)),
|
||||
batch_timeout_ms: env::var(ENV_SINKS_KAFKA_BATCH_TIMEOUT_MS)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Webhook Sink Configuration - Add Retry Parameters
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct WebhookSinkConfig {
|
||||
pub endpoint: String,
|
||||
pub auth_token: String,
|
||||
pub max_retries: Option<usize>, // Maximum number of retry times, default 3
|
||||
pub retry_delay_ms: Option<u64>, // Retry the delay cardinality, default 100ms
|
||||
}
|
||||
|
||||
impl WebhookSinkConfig {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for WebhookSinkConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
endpoint: env::var(ENV_SINKS_WEBHOOK_ENDPOINT)
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_SINKS_WEBHOOK_ENDPOINT.to_string()),
|
||||
auth_token: env::var(ENV_SINKS_WEBHOOK_AUTH_TOKEN)
|
||||
.ok()
|
||||
.filter(|s| !s.trim().is_empty())
|
||||
.unwrap_or_else(|| DEFAULT_SINKS_WEBHOOK_AUTH_TOKEN.to_string()),
|
||||
max_retries: env::var(ENV_SINKS_WEBHOOK_MAX_RETRIES)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_WEBHOOK_MAX_RETRIES)),
|
||||
retry_delay_ms: env::var(ENV_SINKS_WEBHOOK_RETRY_DELAY_MS)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// File Sink Configuration - Add buffering parameters
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct FileSinkConfig {
|
||||
pub path: String,
|
||||
pub buffer_size: Option<usize>, // Write buffer size, default 8192
|
||||
pub flush_interval_ms: Option<u64>, // Refresh interval time, default 1000ms
|
||||
pub flush_threshold: Option<usize>, // Refresh threshold, default 100 logs
|
||||
}
|
||||
|
||||
impl FileSinkConfig {
|
||||
pub fn new() -> Self {
|
||||
Self::default()
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for FileSinkConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
path: get_log_directory_to_string(ENV_SINKS_FILE_PATH),
|
||||
buffer_size: env::var(ENV_SINKS_FILE_BUFFER_SIZE)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_FILE_BUFFER_SIZE)),
|
||||
flush_interval_ms: env::var(ENV_SINKS_FILE_FLUSH_INTERVAL_MS)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS)),
|
||||
flush_threshold: env::var(ENV_SINKS_FILE_FLUSH_THRESHOLD)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_SINKS_FILE_FLUSH_THRESHOLD)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Sink configuration collection
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum SinkConfig {
|
||||
File(FileSinkConfig),
|
||||
Kafka(KafkaSinkConfig),
|
||||
Webhook(WebhookSinkConfig),
|
||||
}
|
||||
|
||||
impl SinkConfig {
|
||||
pub fn new() -> Self {
|
||||
Self::File(FileSinkConfig::new())
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for SinkConfig {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
///Logger Configuration
|
||||
#[derive(Debug, Deserialize, Serialize, Clone)]
|
||||
pub struct LoggerConfig {
|
||||
pub queue_capacity: Option<usize>,
|
||||
}
|
||||
|
||||
impl LoggerConfig {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
queue_capacity: env::var(ENV_AUDIT_LOGGER_QUEUE_CAPACITY)
|
||||
.ok()
|
||||
.and_then(|v| v.parse().ok())
|
||||
.or(Some(DEFAULT_AUDIT_LOGGER_QUEUE_CAPACITY)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for LoggerConfig {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Overall application configuration
|
||||
/// Add observability configuration
|
||||
/// Add observability, sinks, and logger configuration
|
||||
///
|
||||
/// Observability: OpenTelemetry configuration
|
||||
/// Sinks: Kafka, Webhook, File sink configuration
|
||||
/// Logger: Logger configuration
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
@@ -152,6 +316,8 @@ impl Default for OtelConfig {
|
||||
#[derive(Debug, Deserialize, Clone)]
|
||||
pub struct AppConfig {
|
||||
pub observability: OtelConfig,
|
||||
pub sinks: Vec<SinkConfig>,
|
||||
pub logger: Option<LoggerConfig>,
|
||||
}
|
||||
|
||||
impl AppConfig {
|
||||
@@ -162,12 +328,16 @@ impl AppConfig {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
observability: OtelConfig::default(),
|
||||
sinks: vec![SinkConfig::default()],
|
||||
logger: Some(LoggerConfig::default()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn new_with_endpoint(endpoint: Option<String>) -> Self {
|
||||
Self {
|
||||
observability: OtelConfig::extract_otel_config_from_env(endpoint),
|
||||
sinks: vec![SinkConfig::new()],
|
||||
logger: Some(LoggerConfig::new()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
88
crates/obs/src/entry/args.rs
Normal file
88
crates/obs/src/entry/args.rs
Normal file
@@ -0,0 +1,88 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::entry::ObjectVersion;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Args - defines the arguments for API operations
|
||||
/// Args is used to define the arguments for API operations.
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::Args;
|
||||
/// use std::collections::HashMap;
|
||||
///
|
||||
/// let args = Args::new()
|
||||
/// .set_bucket(Some("my-bucket".to_string()))
|
||||
/// .set_object(Some("my-object".to_string()))
|
||||
/// .set_version_id(Some("123".to_string()))
|
||||
/// .set_metadata(Some(HashMap::new()));
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Default, Eq, PartialEq)]
|
||||
pub struct Args {
|
||||
#[serde(rename = "bucket", skip_serializing_if = "Option::is_none")]
|
||||
pub bucket: Option<String>,
|
||||
#[serde(rename = "object", skip_serializing_if = "Option::is_none")]
|
||||
pub object: Option<String>,
|
||||
#[serde(rename = "versionId", skip_serializing_if = "Option::is_none")]
|
||||
pub version_id: Option<String>,
|
||||
#[serde(rename = "objects", skip_serializing_if = "Option::is_none")]
|
||||
pub objects: Option<Vec<ObjectVersion>>,
|
||||
#[serde(rename = "metadata", skip_serializing_if = "Option::is_none")]
|
||||
pub metadata: Option<HashMap<String, String>>,
|
||||
}
|
||||
|
||||
impl Args {
|
||||
/// Create a new Args object
|
||||
pub fn new() -> Self {
|
||||
Args {
|
||||
bucket: None,
|
||||
object: None,
|
||||
version_id: None,
|
||||
objects: None,
|
||||
metadata: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the bucket
|
||||
pub fn set_bucket(mut self, bucket: Option<String>) -> Self {
|
||||
self.bucket = bucket;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object
|
||||
pub fn set_object(mut self, object: Option<String>) -> Self {
|
||||
self.object = object;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the version ID
|
||||
pub fn set_version_id(mut self, version_id: Option<String>) -> Self {
|
||||
self.version_id = version_id;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the objects
|
||||
pub fn set_objects(mut self, objects: Option<Vec<ObjectVersion>>) -> Self {
|
||||
self.objects = objects;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the metadata
|
||||
pub fn set_metadata(mut self, metadata: Option<HashMap<String, String>>) -> Self {
|
||||
self.metadata = metadata;
|
||||
self
|
||||
}
|
||||
}
|
||||
467
crates/obs/src/entry/audit.rs
Normal file
467
crates/obs/src/entry/audit.rs
Normal file
@@ -0,0 +1,467 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{BaseLogEntry, LogRecord, ObjectVersion};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// API details structure
|
||||
/// ApiDetails is used to define the details of an API operation
|
||||
///
|
||||
/// The `ApiDetails` structure contains the following fields:
|
||||
/// - `name` - the name of the API operation
|
||||
/// - `bucket` - the bucket name
|
||||
/// - `object` - the object name
|
||||
/// - `objects` - the list of objects
|
||||
/// - `status` - the status of the API operation
|
||||
/// - `status_code` - the status code of the API operation
|
||||
/// - `input_bytes` - the input bytes
|
||||
/// - `output_bytes` - the output bytes
|
||||
/// - `header_bytes` - the header bytes
|
||||
/// - `time_to_first_byte` - the time to first byte
|
||||
/// - `time_to_first_byte_in_ns` - the time to first byte in nanoseconds
|
||||
/// - `time_to_response` - the time to response
|
||||
/// - `time_to_response_in_ns` - the time to response in nanoseconds
|
||||
///
|
||||
/// The `ApiDetails` structure contains the following methods:
|
||||
/// - `new` - create a new `ApiDetails` with default values
|
||||
/// - `set_name` - set the name
|
||||
/// - `set_bucket` - set the bucket
|
||||
/// - `set_object` - set the object
|
||||
/// - `set_objects` - set the objects
|
||||
/// - `set_status` - set the status
|
||||
/// - `set_status_code` - set the status code
|
||||
/// - `set_input_bytes` - set the input bytes
|
||||
/// - `set_output_bytes` - set the output bytes
|
||||
/// - `set_header_bytes` - set the header bytes
|
||||
/// - `set_time_to_first_byte` - set the time to first byte
|
||||
/// - `set_time_to_first_byte_in_ns` - set the time to first byte in nanoseconds
|
||||
/// - `set_time_to_response` - set the time to response
|
||||
/// - `set_time_to_response_in_ns` - set the time to response in nanoseconds
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::ApiDetails;
|
||||
/// use rustfs_obs::ObjectVersion;
|
||||
///
|
||||
/// let api = ApiDetails::new()
|
||||
/// .set_name(Some("GET".to_string()))
|
||||
/// .set_bucket(Some("my-bucket".to_string()))
|
||||
/// .set_object(Some("my-object".to_string()))
|
||||
/// .set_objects(vec![ObjectVersion::new_with_object_name("my-object".to_string())])
|
||||
/// .set_status(Some("OK".to_string()))
|
||||
/// .set_status_code(Some(200))
|
||||
/// .set_input_bytes(100)
|
||||
/// .set_output_bytes(200)
|
||||
/// .set_header_bytes(Some(50))
|
||||
/// .set_time_to_first_byte(Some("100ms".to_string()))
|
||||
/// .set_time_to_first_byte_in_ns(Some("100000000ns".to_string()))
|
||||
/// .set_time_to_response(Some("200ms".to_string()))
|
||||
/// .set_time_to_response_in_ns(Some("200000000ns".to_string()));
|
||||
/// ```
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, Default, PartialEq, Eq)]
|
||||
pub struct ApiDetails {
|
||||
#[serde(rename = "name", skip_serializing_if = "Option::is_none")]
|
||||
pub name: Option<String>,
|
||||
#[serde(rename = "bucket", skip_serializing_if = "Option::is_none")]
|
||||
pub bucket: Option<String>,
|
||||
#[serde(rename = "object", skip_serializing_if = "Option::is_none")]
|
||||
pub object: Option<String>,
|
||||
#[serde(rename = "objects", skip_serializing_if = "Vec::is_empty", default)]
|
||||
pub objects: Vec<ObjectVersion>,
|
||||
#[serde(rename = "status", skip_serializing_if = "Option::is_none")]
|
||||
pub status: Option<String>,
|
||||
#[serde(rename = "statusCode", skip_serializing_if = "Option::is_none")]
|
||||
pub status_code: Option<i32>,
|
||||
#[serde(rename = "rx")]
|
||||
pub input_bytes: i64,
|
||||
#[serde(rename = "tx")]
|
||||
pub output_bytes: i64,
|
||||
#[serde(rename = "txHeaders", skip_serializing_if = "Option::is_none")]
|
||||
pub header_bytes: Option<i64>,
|
||||
#[serde(rename = "timeToFirstByte", skip_serializing_if = "Option::is_none")]
|
||||
pub time_to_first_byte: Option<String>,
|
||||
#[serde(rename = "timeToFirstByteInNS", skip_serializing_if = "Option::is_none")]
|
||||
pub time_to_first_byte_in_ns: Option<String>,
|
||||
#[serde(rename = "timeToResponse", skip_serializing_if = "Option::is_none")]
|
||||
pub time_to_response: Option<String>,
|
||||
#[serde(rename = "timeToResponseInNS", skip_serializing_if = "Option::is_none")]
|
||||
pub time_to_response_in_ns: Option<String>,
|
||||
}
|
||||
|
||||
impl ApiDetails {
|
||||
/// Create a new `ApiDetails` with default values
|
||||
pub fn new() -> Self {
|
||||
ApiDetails {
|
||||
name: None,
|
||||
bucket: None,
|
||||
object: None,
|
||||
objects: Vec::new(),
|
||||
status: None,
|
||||
status_code: None,
|
||||
input_bytes: 0,
|
||||
output_bytes: 0,
|
||||
header_bytes: None,
|
||||
time_to_first_byte: None,
|
||||
time_to_first_byte_in_ns: None,
|
||||
time_to_response: None,
|
||||
time_to_response_in_ns: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the name
|
||||
pub fn set_name(mut self, name: Option<String>) -> Self {
|
||||
self.name = name;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the bucket
|
||||
pub fn set_bucket(mut self, bucket: Option<String>) -> Self {
|
||||
self.bucket = bucket;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the object
|
||||
pub fn set_object(mut self, object: Option<String>) -> Self {
|
||||
self.object = object;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the objects
|
||||
pub fn set_objects(mut self, objects: Vec<ObjectVersion>) -> Self {
|
||||
self.objects = objects;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the status
|
||||
pub fn set_status(mut self, status: Option<String>) -> Self {
|
||||
self.status = status;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the status code
|
||||
pub fn set_status_code(mut self, status_code: Option<i32>) -> Self {
|
||||
self.status_code = status_code;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the input bytes
|
||||
pub fn set_input_bytes(mut self, input_bytes: i64) -> Self {
|
||||
self.input_bytes = input_bytes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the output bytes
|
||||
pub fn set_output_bytes(mut self, output_bytes: i64) -> Self {
|
||||
self.output_bytes = output_bytes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the header bytes
|
||||
pub fn set_header_bytes(mut self, header_bytes: Option<i64>) -> Self {
|
||||
self.header_bytes = header_bytes;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the time to first byte
|
||||
pub fn set_time_to_first_byte(mut self, time_to_first_byte: Option<String>) -> Self {
|
||||
self.time_to_first_byte = time_to_first_byte;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the time to first byte in nanoseconds
|
||||
pub fn set_time_to_first_byte_in_ns(mut self, time_to_first_byte_in_ns: Option<String>) -> Self {
|
||||
self.time_to_first_byte_in_ns = time_to_first_byte_in_ns;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the time to response
|
||||
pub fn set_time_to_response(mut self, time_to_response: Option<String>) -> Self {
|
||||
self.time_to_response = time_to_response;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the time to response in nanoseconds
|
||||
pub fn set_time_to_response_in_ns(mut self, time_to_response_in_ns: Option<String>) -> Self {
|
||||
self.time_to_response_in_ns = time_to_response_in_ns;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Entry - audit entry logs
|
||||
/// AuditLogEntry is used to define the structure of an audit log entry
|
||||
///
|
||||
/// The `AuditLogEntry` structure contains the following fields:
|
||||
/// - `base` - the base log entry
|
||||
/// - `version` - the version of the audit log entry
|
||||
/// - `deployment_id` - the deployment ID
|
||||
/// - `event` - the event
|
||||
/// - `entry_type` - the type of audit message
|
||||
/// - `api` - the API details
|
||||
/// - `remote_host` - the remote host
|
||||
/// - `user_agent` - the user agent
|
||||
/// - `req_path` - the request path
|
||||
/// - `req_host` - the request host
|
||||
/// - `req_claims` - the request claims
|
||||
/// - `req_query` - the request query
|
||||
/// - `req_header` - the request header
|
||||
/// - `resp_header` - the response header
|
||||
/// - `access_key` - the access key
|
||||
/// - `parent_user` - the parent user
|
||||
/// - `error` - the error
|
||||
///
|
||||
/// The `AuditLogEntry` structure contains the following methods:
|
||||
/// - `new` - create a new `AuditEntry` with default values
|
||||
/// - `new_with_values` - create a new `AuditEntry` with version, time, event and api details
|
||||
/// - `with_base` - set the base log entry
|
||||
/// - `set_version` - set the version
|
||||
/// - `set_deployment_id` - set the deployment ID
|
||||
/// - `set_event` - set the event
|
||||
/// - `set_entry_type` - set the entry type
|
||||
/// - `set_api` - set the API details
|
||||
/// - `set_remote_host` - set the remote host
|
||||
/// - `set_user_agent` - set the user agent
|
||||
/// - `set_req_path` - set the request path
|
||||
/// - `set_req_host` - set the request host
|
||||
/// - `set_req_claims` - set the request claims
|
||||
/// - `set_req_query` - set the request query
|
||||
/// - `set_req_header` - set the request header
|
||||
/// - `set_resp_header` - set the response header
|
||||
/// - `set_access_key` - set the access key
|
||||
/// - `set_parent_user` - set the parent user
|
||||
/// - `set_error` - set the error
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::AuditLogEntry;
|
||||
/// use rustfs_obs::ApiDetails;
|
||||
/// use std::collections::HashMap;
|
||||
///
|
||||
/// let entry = AuditLogEntry::new()
|
||||
/// .set_version("1.0".to_string())
|
||||
/// .set_deployment_id(Some("123".to_string()))
|
||||
/// .set_event("event".to_string())
|
||||
/// .set_entry_type(Some("type".to_string()))
|
||||
/// .set_api(ApiDetails::new())
|
||||
/// .set_remote_host(Some("remote-host".to_string()))
|
||||
/// .set_user_agent(Some("user-agent".to_string()))
|
||||
/// .set_req_path(Some("req-path".to_string()))
|
||||
/// .set_req_host(Some("req-host".to_string()))
|
||||
/// .set_req_claims(Some(HashMap::new()))
|
||||
/// .set_req_query(Some(HashMap::new()))
|
||||
/// .set_req_header(Some(HashMap::new()))
|
||||
/// .set_resp_header(Some(HashMap::new()))
|
||||
/// .set_access_key(Some("access-key".to_string()))
|
||||
/// .set_parent_user(Some("parent-user".to_string()))
|
||||
/// .set_error(Some("error".to_string()));
|
||||
#[derive(Debug, Serialize, Deserialize, Clone, Default)]
|
||||
pub struct AuditLogEntry {
|
||||
#[serde(flatten)]
|
||||
pub base: BaseLogEntry,
|
||||
pub version: String,
|
||||
#[serde(rename = "deploymentid", skip_serializing_if = "Option::is_none")]
|
||||
pub deployment_id: Option<String>,
|
||||
pub event: String,
|
||||
// Class of audit message - S3, admin ops, bucket management
|
||||
#[serde(rename = "type", skip_serializing_if = "Option::is_none")]
|
||||
pub entry_type: Option<String>,
|
||||
pub api: ApiDetails,
|
||||
#[serde(rename = "remotehost", skip_serializing_if = "Option::is_none")]
|
||||
pub remote_host: Option<String>,
|
||||
#[serde(rename = "userAgent", skip_serializing_if = "Option::is_none")]
|
||||
pub user_agent: Option<String>,
|
||||
#[serde(rename = "requestPath", skip_serializing_if = "Option::is_none")]
|
||||
pub req_path: Option<String>,
|
||||
#[serde(rename = "requestHost", skip_serializing_if = "Option::is_none")]
|
||||
pub req_host: Option<String>,
|
||||
#[serde(rename = "requestClaims", skip_serializing_if = "Option::is_none")]
|
||||
pub req_claims: Option<HashMap<String, Value>>,
|
||||
#[serde(rename = "requestQuery", skip_serializing_if = "Option::is_none")]
|
||||
pub req_query: Option<HashMap<String, String>>,
|
||||
#[serde(rename = "requestHeader", skip_serializing_if = "Option::is_none")]
|
||||
pub req_header: Option<HashMap<String, String>>,
|
||||
#[serde(rename = "responseHeader", skip_serializing_if = "Option::is_none")]
|
||||
pub resp_header: Option<HashMap<String, String>>,
|
||||
#[serde(rename = "accessKey", skip_serializing_if = "Option::is_none")]
|
||||
pub access_key: Option<String>,
|
||||
#[serde(rename = "parentUser", skip_serializing_if = "Option::is_none")]
|
||||
pub parent_user: Option<String>,
|
||||
#[serde(rename = "error", skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
impl AuditLogEntry {
|
||||
/// Create a new `AuditEntry` with default values
|
||||
pub fn new() -> Self {
|
||||
AuditLogEntry {
|
||||
base: BaseLogEntry::new(),
|
||||
version: String::new(),
|
||||
deployment_id: None,
|
||||
event: String::new(),
|
||||
entry_type: None,
|
||||
api: ApiDetails::new(),
|
||||
remote_host: None,
|
||||
user_agent: None,
|
||||
req_path: None,
|
||||
req_host: None,
|
||||
req_claims: None,
|
||||
req_query: None,
|
||||
req_header: None,
|
||||
resp_header: None,
|
||||
access_key: None,
|
||||
parent_user: None,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new `AuditEntry` with version, time, event and api details
|
||||
pub fn new_with_values(version: String, time: DateTime<Utc>, event: String, api: ApiDetails) -> Self {
|
||||
let mut base = BaseLogEntry::new();
|
||||
base.timestamp = time;
|
||||
|
||||
AuditLogEntry {
|
||||
base,
|
||||
version,
|
||||
deployment_id: None,
|
||||
event,
|
||||
entry_type: None,
|
||||
api,
|
||||
remote_host: None,
|
||||
user_agent: None,
|
||||
req_path: None,
|
||||
req_host: None,
|
||||
req_claims: None,
|
||||
req_query: None,
|
||||
req_header: None,
|
||||
resp_header: None,
|
||||
access_key: None,
|
||||
parent_user: None,
|
||||
error: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the base log entry
|
||||
pub fn with_base(mut self, base: BaseLogEntry) -> Self {
|
||||
self.base = base;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the version
|
||||
pub fn set_version(mut self, version: String) -> Self {
|
||||
self.version = version;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the deployment ID
|
||||
pub fn set_deployment_id(mut self, deployment_id: Option<String>) -> Self {
|
||||
self.deployment_id = deployment_id;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the event
|
||||
pub fn set_event(mut self, event: String) -> Self {
|
||||
self.event = event;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the entry type
|
||||
pub fn set_entry_type(mut self, entry_type: Option<String>) -> Self {
|
||||
self.entry_type = entry_type;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the API details
|
||||
pub fn set_api(mut self, api: ApiDetails) -> Self {
|
||||
self.api = api;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the remote host
|
||||
pub fn set_remote_host(mut self, remote_host: Option<String>) -> Self {
|
||||
self.remote_host = remote_host;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the user agent
|
||||
pub fn set_user_agent(mut self, user_agent: Option<String>) -> Self {
|
||||
self.user_agent = user_agent;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request path
|
||||
pub fn set_req_path(mut self, req_path: Option<String>) -> Self {
|
||||
self.req_path = req_path;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request host
|
||||
pub fn set_req_host(mut self, req_host: Option<String>) -> Self {
|
||||
self.req_host = req_host;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request claims
|
||||
pub fn set_req_claims(mut self, req_claims: Option<HashMap<String, Value>>) -> Self {
|
||||
self.req_claims = req_claims;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request query
|
||||
pub fn set_req_query(mut self, req_query: Option<HashMap<String, String>>) -> Self {
|
||||
self.req_query = req_query;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request header
|
||||
pub fn set_req_header(mut self, req_header: Option<HashMap<String, String>>) -> Self {
|
||||
self.req_header = req_header;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the response header
|
||||
pub fn set_resp_header(mut self, resp_header: Option<HashMap<String, String>>) -> Self {
|
||||
self.resp_header = resp_header;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the access key
|
||||
pub fn set_access_key(mut self, access_key: Option<String>) -> Self {
|
||||
self.access_key = access_key;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the parent user
|
||||
pub fn set_parent_user(mut self, parent_user: Option<String>) -> Self {
|
||||
self.parent_user = parent_user;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the error
|
||||
pub fn set_error(mut self, error: Option<String>) -> Self {
|
||||
self.error = error;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl LogRecord for AuditLogEntry {
|
||||
fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_else(|_| String::from("{}"))
|
||||
}
|
||||
|
||||
fn get_timestamp(&self) -> DateTime<Utc> {
|
||||
self.base.timestamp
|
||||
}
|
||||
}
|
||||
106
crates/obs/src/entry/base.rs
Normal file
106
crates/obs/src/entry/base.rs
Normal file
@@ -0,0 +1,106 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::HashMap;
|
||||
|
||||
/// Base log entry structure shared by all log types
|
||||
/// This structure is used to serialize log entries to JSON
|
||||
/// and send them to the log sinks
|
||||
/// This structure is also used to deserialize log entries from JSON
|
||||
/// This structure is also used to store log entries in the database
|
||||
/// This structure is also used to query log entries from the database
|
||||
///
|
||||
/// The `BaseLogEntry` structure contains the following fields:
|
||||
/// - `timestamp` - the timestamp of the log entry
|
||||
/// - `request_id` - the request ID of the log entry
|
||||
/// - `message` - the message of the log entry
|
||||
/// - `tags` - the tags of the log entry
|
||||
///
|
||||
/// The `BaseLogEntry` structure contains the following methods:
|
||||
/// - `new` - create a new `BaseLogEntry` with default values
|
||||
/// - `message` - set the message
|
||||
/// - `request_id` - set the request ID
|
||||
/// - `tags` - set the tags
|
||||
/// - `timestamp` - set the timestamp
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::BaseLogEntry;
|
||||
/// use chrono::{DateTime, Utc};
|
||||
/// use std::collections::HashMap;
|
||||
///
|
||||
/// let timestamp = Utc::now();
|
||||
/// let request = Some("req-123".to_string());
|
||||
/// let message = Some("This is a log message".to_string());
|
||||
/// let tags = Some(HashMap::new());
|
||||
///
|
||||
/// let entry = BaseLogEntry::new()
|
||||
/// .timestamp(timestamp)
|
||||
/// .request_id(request)
|
||||
/// .message(message)
|
||||
/// .tags(tags);
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq, Default)]
|
||||
pub struct BaseLogEntry {
|
||||
#[serde(rename = "time")]
|
||||
pub timestamp: DateTime<Utc>,
|
||||
|
||||
#[serde(rename = "requestID", skip_serializing_if = "Option::is_none")]
|
||||
pub request_id: Option<String>,
|
||||
|
||||
#[serde(rename = "message", skip_serializing_if = "Option::is_none")]
|
||||
pub message: Option<String>,
|
||||
|
||||
#[serde(rename = "tags", skip_serializing_if = "Option::is_none")]
|
||||
pub tags: Option<HashMap<String, Value>>,
|
||||
}
|
||||
|
||||
impl BaseLogEntry {
|
||||
/// Create a new BaseLogEntry with default values
|
||||
pub fn new() -> Self {
|
||||
BaseLogEntry {
|
||||
timestamp: Utc::now(),
|
||||
request_id: None,
|
||||
message: None,
|
||||
tags: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the message
|
||||
pub fn message(mut self, message: Option<String>) -> Self {
|
||||
self.message = message;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the request ID
|
||||
pub fn request_id(mut self, request_id: Option<String>) -> Self {
|
||||
self.request_id = request_id;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the tags
|
||||
pub fn tags(mut self, tags: Option<HashMap<String, Value>>) -> Self {
|
||||
self.tags = tags;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the timestamp
|
||||
pub fn timestamp(mut self, timestamp: DateTime<Utc>) -> Self {
|
||||
self.timestamp = timestamp;
|
||||
self
|
||||
}
|
||||
}
|
||||
158
crates/obs/src/entry/mod.rs
Normal file
158
crates/obs/src/entry/mod.rs
Normal file
@@ -0,0 +1,158 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
pub(crate) mod args;
|
||||
pub(crate) mod audit;
|
||||
pub(crate) mod base;
|
||||
pub(crate) mod unified;
|
||||
|
||||
use serde::de::Error;
|
||||
use serde::{Deserialize, Deserializer, Serialize, Serializer};
|
||||
use tracing_core::Level;
|
||||
|
||||
/// ObjectVersion is used across multiple modules
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, Eq, PartialEq)]
|
||||
pub struct ObjectVersion {
|
||||
#[serde(rename = "name")]
|
||||
pub object_name: String,
|
||||
#[serde(rename = "versionId", skip_serializing_if = "Option::is_none")]
|
||||
pub version_id: Option<String>,
|
||||
}
|
||||
|
||||
impl ObjectVersion {
|
||||
/// Create a new ObjectVersion object
|
||||
pub fn new() -> Self {
|
||||
ObjectVersion {
|
||||
object_name: String::new(),
|
||||
version_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new ObjectVersion with object name
|
||||
pub fn new_with_object_name(object_name: String) -> Self {
|
||||
ObjectVersion {
|
||||
object_name,
|
||||
version_id: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the object name
|
||||
pub fn set_object_name(mut self, object_name: String) -> Self {
|
||||
self.object_name = object_name;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the version ID
|
||||
pub fn set_version_id(mut self, version_id: Option<String>) -> Self {
|
||||
self.version_id = version_id;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ObjectVersion {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
/// Log kind/level enum
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq, Default)]
|
||||
pub enum LogKind {
|
||||
#[serde(rename = "INFO")]
|
||||
#[default]
|
||||
Info,
|
||||
#[serde(rename = "WARNING")]
|
||||
Warning,
|
||||
#[serde(rename = "ERROR")]
|
||||
Error,
|
||||
#[serde(rename = "FATAL")]
|
||||
Fatal,
|
||||
}
|
||||
|
||||
/// Trait for types that can be serialized to JSON and have a timestamp
|
||||
/// This trait is used by `ServerLogEntry` to convert the log entry to JSON
|
||||
/// and get the timestamp of the log entry
|
||||
/// This trait is implemented by `ServerLogEntry`
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::LogRecord;
|
||||
/// use chrono::{DateTime, Utc};
|
||||
/// use rustfs_obs::ServerLogEntry;
|
||||
/// use tracing_core::Level;
|
||||
///
|
||||
/// let log_entry = ServerLogEntry::new(Level::INFO, "api_handler".to_string());
|
||||
/// let json = log_entry.to_json();
|
||||
/// let timestamp = log_entry.get_timestamp();
|
||||
/// ```
|
||||
pub trait LogRecord {
|
||||
fn to_json(&self) -> String;
|
||||
fn get_timestamp(&self) -> chrono::DateTime<chrono::Utc>;
|
||||
}
|
||||
|
||||
/// Wrapper for `tracing_core::Level` to implement `Serialize` and `Deserialize`
|
||||
/// for `ServerLogEntry`
|
||||
/// This is necessary because `tracing_core::Level` does not implement `Serialize`
|
||||
/// and `Deserialize`
|
||||
/// This is a workaround to allow `ServerLogEntry` to be serialized and deserialized
|
||||
/// using `serde`
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::SerializableLevel;
|
||||
/// use tracing_core::Level;
|
||||
///
|
||||
/// let level = Level::INFO;
|
||||
/// let serializable_level = SerializableLevel::from(level);
|
||||
/// ```
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SerializableLevel(pub Level);
|
||||
|
||||
impl From<Level> for SerializableLevel {
|
||||
fn from(level: Level) -> Self {
|
||||
SerializableLevel(level)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<SerializableLevel> for Level {
|
||||
fn from(serializable_level: SerializableLevel) -> Self {
|
||||
serializable_level.0
|
||||
}
|
||||
}
|
||||
|
||||
impl Serialize for SerializableLevel {
|
||||
fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
|
||||
where
|
||||
S: Serializer,
|
||||
{
|
||||
serializer.serialize_str(self.0.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
impl<'de> Deserialize<'de> for SerializableLevel {
|
||||
fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
|
||||
where
|
||||
D: Deserializer<'de>,
|
||||
{
|
||||
let s = String::deserialize(deserializer)?;
|
||||
match s.as_str() {
|
||||
"TRACE" => Ok(SerializableLevel(Level::TRACE)),
|
||||
"DEBUG" => Ok(SerializableLevel(Level::DEBUG)),
|
||||
"INFO" => Ok(SerializableLevel(Level::INFO)),
|
||||
"WARN" => Ok(SerializableLevel(Level::WARN)),
|
||||
"ERROR" => Ok(SerializableLevel(Level::ERROR)),
|
||||
_ => Err(D::Error::custom("unknown log level")),
|
||||
}
|
||||
}
|
||||
}
|
||||
301
crates/obs/src/entry/unified.rs
Normal file
301
crates/obs/src/entry/unified.rs
Normal file
@@ -0,0 +1,301 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{AuditLogEntry, BaseLogEntry, LogKind, LogRecord, SerializableLevel};
|
||||
use chrono::{DateTime, Utc};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing_core::Level;
|
||||
|
||||
/// Server log entry with structured fields
|
||||
/// ServerLogEntry is used to log structured log entries from the server
|
||||
///
|
||||
/// The `ServerLogEntry` structure contains the following fields:
|
||||
/// - `base` - the base log entry
|
||||
/// - `level` - the log level
|
||||
/// - `source` - the source of the log entry
|
||||
/// - `user_id` - the user ID
|
||||
/// - `fields` - the structured fields of the log entry
|
||||
///
|
||||
/// The `ServerLogEntry` structure contains the following methods:
|
||||
/// - `new` - create a new `ServerLogEntry` with specified level and source
|
||||
/// - `with_base` - set the base log entry
|
||||
/// - `user_id` - set the user ID
|
||||
/// - `fields` - set the fields
|
||||
/// - `add_field` - add a field
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::ServerLogEntry;
|
||||
/// use tracing_core::Level;
|
||||
///
|
||||
/// let entry = ServerLogEntry::new(Level::INFO, "test_module".to_string())
|
||||
/// .user_id(Some("user-456".to_string()))
|
||||
/// .add_field("operation".to_string(), "login".to_string());
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
|
||||
pub struct ServerLogEntry {
|
||||
#[serde(flatten)]
|
||||
pub base: BaseLogEntry,
|
||||
|
||||
pub level: SerializableLevel,
|
||||
pub source: String,
|
||||
|
||||
#[serde(rename = "userId", skip_serializing_if = "Option::is_none")]
|
||||
pub user_id: Option<String>,
|
||||
|
||||
#[serde(skip_serializing_if = "Vec::is_empty", default)]
|
||||
pub fields: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl ServerLogEntry {
|
||||
/// Create a new ServerLogEntry with specified level and source
|
||||
pub fn new(level: Level, source: String) -> Self {
|
||||
ServerLogEntry {
|
||||
base: BaseLogEntry::new(),
|
||||
level: SerializableLevel(level),
|
||||
source,
|
||||
user_id: None,
|
||||
fields: Vec::new(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the base log entry
|
||||
pub fn with_base(mut self, base: BaseLogEntry) -> Self {
|
||||
self.base = base;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the user ID
|
||||
pub fn user_id(mut self, user_id: Option<String>) -> Self {
|
||||
self.user_id = user_id;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set fields
|
||||
pub fn fields(mut self, fields: Vec<(String, String)>) -> Self {
|
||||
self.fields = fields;
|
||||
self
|
||||
}
|
||||
|
||||
/// Add a field
|
||||
pub fn add_field(mut self, key: String, value: String) -> Self {
|
||||
self.fields.push((key, value));
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl LogRecord for ServerLogEntry {
|
||||
fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_else(|_| String::from("{}"))
|
||||
}
|
||||
|
||||
fn get_timestamp(&self) -> DateTime<Utc> {
|
||||
self.base.timestamp
|
||||
}
|
||||
}
|
||||
|
||||
/// Console log entry structure
|
||||
/// ConsoleLogEntry is used to log console log entries
|
||||
/// The `ConsoleLogEntry` structure contains the following fields:
|
||||
/// - `base` - the base log entry
|
||||
/// - `level` - the log level
|
||||
/// - `console_msg` - the console message
|
||||
/// - `node_name` - the node name
|
||||
/// - `err` - the error message
|
||||
///
|
||||
/// The `ConsoleLogEntry` structure contains the following methods:
|
||||
/// - `new` - create a new `ConsoleLogEntry`
|
||||
/// - `new_with_console_msg` - create a new `ConsoleLogEntry` with console message and node name
|
||||
/// - `with_base` - set the base log entry
|
||||
/// - `set_level` - set the log level
|
||||
/// - `set_node_name` - set the node name
|
||||
/// - `set_console_msg` - set the console message
|
||||
/// - `set_err` - set the error message
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::ConsoleLogEntry;
|
||||
///
|
||||
/// let entry = ConsoleLogEntry::new_with_console_msg("Test message".to_string(), "node-123".to_string());
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct ConsoleLogEntry {
|
||||
#[serde(flatten)]
|
||||
pub base: BaseLogEntry,
|
||||
|
||||
pub level: LogKind,
|
||||
pub console_msg: String,
|
||||
pub node_name: String,
|
||||
|
||||
#[serde(skip)]
|
||||
pub err: Option<String>,
|
||||
}
|
||||
|
||||
impl ConsoleLogEntry {
|
||||
/// Create a new ConsoleLogEntry
|
||||
pub fn new() -> Self {
|
||||
ConsoleLogEntry {
|
||||
base: BaseLogEntry::new(),
|
||||
level: LogKind::Info,
|
||||
console_msg: String::new(),
|
||||
node_name: String::new(),
|
||||
err: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new ConsoleLogEntry with console message and node name
|
||||
pub fn new_with_console_msg(console_msg: String, node_name: String) -> Self {
|
||||
ConsoleLogEntry {
|
||||
base: BaseLogEntry::new(),
|
||||
level: LogKind::Info,
|
||||
console_msg,
|
||||
node_name,
|
||||
err: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the base log entry
|
||||
pub fn with_base(mut self, base: BaseLogEntry) -> Self {
|
||||
self.base = base;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the log level
|
||||
pub fn set_level(mut self, level: LogKind) -> Self {
|
||||
self.level = level;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the node name
|
||||
pub fn set_node_name(mut self, node_name: String) -> Self {
|
||||
self.node_name = node_name;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the console message
|
||||
pub fn set_console_msg(mut self, console_msg: String) -> Self {
|
||||
self.console_msg = console_msg;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the error message
|
||||
pub fn set_err(mut self, err: Option<String>) -> Self {
|
||||
self.err = err;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for ConsoleLogEntry {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl LogRecord for ConsoleLogEntry {
|
||||
fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_else(|_| String::from("{}"))
|
||||
}
|
||||
|
||||
fn get_timestamp(&self) -> DateTime<Utc> {
|
||||
self.base.timestamp
|
||||
}
|
||||
}
|
||||
|
||||
/// Unified log entry type
|
||||
/// UnifiedLogEntry is used to log different types of log entries
|
||||
///
|
||||
/// The `UnifiedLogEntry` enum contains the following variants:
|
||||
/// - `Server` - a server log entry
|
||||
/// - `Audit` - an audit log entry
|
||||
/// - `Console` - a console log entry
|
||||
///
|
||||
/// The `UnifiedLogEntry` enum contains the following methods:
|
||||
/// - `to_json` - convert the log entry to JSON
|
||||
/// - `get_timestamp` - get the timestamp of the log entry
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::{UnifiedLogEntry, ServerLogEntry};
|
||||
/// use tracing_core::Level;
|
||||
///
|
||||
/// let server_entry = ServerLogEntry::new(Level::INFO, "test_module".to_string());
|
||||
/// let unified = UnifiedLogEntry::Server(server_entry);
|
||||
/// ```
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
#[serde(tag = "type")]
|
||||
pub enum UnifiedLogEntry {
|
||||
#[serde(rename = "server")]
|
||||
Server(ServerLogEntry),
|
||||
|
||||
#[serde(rename = "audit")]
|
||||
Audit(Box<AuditLogEntry>),
|
||||
|
||||
#[serde(rename = "console")]
|
||||
Console(ConsoleLogEntry),
|
||||
}
|
||||
|
||||
impl LogRecord for UnifiedLogEntry {
|
||||
fn to_json(&self) -> String {
|
||||
match self {
|
||||
UnifiedLogEntry::Server(entry) => entry.to_json(),
|
||||
UnifiedLogEntry::Audit(entry) => entry.to_json(),
|
||||
UnifiedLogEntry::Console(entry) => entry.to_json(),
|
||||
}
|
||||
}
|
||||
|
||||
fn get_timestamp(&self) -> DateTime<Utc> {
|
||||
match self {
|
||||
UnifiedLogEntry::Server(entry) => entry.get_timestamp(),
|
||||
UnifiedLogEntry::Audit(entry) => entry.get_timestamp(),
|
||||
UnifiedLogEntry::Console(entry) => entry.get_timestamp(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_base_log_entry() {
|
||||
let base = BaseLogEntry::new()
|
||||
.request_id(Some("req-123".to_string()))
|
||||
.message(Some("Test message".to_string()));
|
||||
|
||||
assert_eq!(base.request_id, Some("req-123".to_string()));
|
||||
assert_eq!(base.message, Some("Test message".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_server_log_entry() {
|
||||
let entry = ServerLogEntry::new(Level::INFO, "test_module".to_string())
|
||||
.user_id(Some("user-456".to_string()))
|
||||
.add_field("operation".to_string(), "login".to_string());
|
||||
|
||||
assert_eq!(entry.level.0, Level::INFO);
|
||||
assert_eq!(entry.source, "test_module");
|
||||
assert_eq!(entry.user_id, Some("user-456".to_string()));
|
||||
assert_eq!(entry.fields.len(), 1);
|
||||
assert_eq!(entry.fields[0], ("operation".to_string(), "login".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_unified_log_entry_json() {
|
||||
let server_entry = ServerLogEntry::new(Level::INFO, "test_source".to_string());
|
||||
let unified = UnifiedLogEntry::Server(server_entry);
|
||||
|
||||
let json = unified.to_json();
|
||||
assert!(json.contains("test_source"));
|
||||
}
|
||||
}
|
||||
@@ -12,10 +12,9 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::AppConfig;
|
||||
use crate::logger::InitLogStatus;
|
||||
use crate::telemetry::{OtelGuard, init_telemetry};
|
||||
use opentelemetry::metrics::Meter;
|
||||
use rustfs_config::APP_NAME;
|
||||
use crate::{AppConfig, Logger, get_global_logger, init_global_logger};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use tokio::sync::{OnceCell, SetError};
|
||||
use tracing::{error, info};
|
||||
@@ -23,23 +22,6 @@ use tracing::{error, info};
|
||||
/// Global guard for OpenTelemetry tracing
|
||||
static GLOBAL_GUARD: OnceCell<Arc<Mutex<OtelGuard>>> = OnceCell::const_new();
|
||||
|
||||
/// Flag indicating if observability is enabled
|
||||
pub(crate) static IS_OBSERVABILITY_ENABLED: OnceCell<bool> = OnceCell::const_new();
|
||||
|
||||
/// Name of the observability meter
|
||||
pub(crate) static OBSERVABILITY_METER_NAME: OnceCell<String> = OnceCell::const_new();
|
||||
|
||||
/// Check whether Observability is enabled
|
||||
pub fn is_observability_enabled() -> bool {
|
||||
IS_OBSERVABILITY_ENABLED.get().copied().unwrap_or(false)
|
||||
}
|
||||
|
||||
/// Get the global meter for observability
|
||||
pub fn global_meter() -> Meter {
|
||||
let meter_name = OBSERVABILITY_METER_NAME.get().map(|s| s.as_str()).unwrap_or(APP_NAME);
|
||||
opentelemetry::global::meter(meter_name)
|
||||
}
|
||||
|
||||
/// Error type for global guard operations
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
pub enum GlobalError {
|
||||
@@ -79,14 +61,46 @@ pub enum GlobalError {
|
||||
///
|
||||
/// # #[tokio::main]
|
||||
/// # async fn main() {
|
||||
/// let guard = init_obs(None).await;
|
||||
/// let (logger, guard) = init_obs(None).await;
|
||||
/// # }
|
||||
/// ```
|
||||
pub async fn init_obs(endpoint: Option<String>) -> OtelGuard {
|
||||
pub async fn init_obs(endpoint: Option<String>) -> (Arc<tokio::sync::Mutex<Logger>>, OtelGuard) {
|
||||
// Load the configuration file
|
||||
let config = AppConfig::new_with_endpoint(endpoint);
|
||||
|
||||
init_telemetry(&config.observability)
|
||||
let guard = init_telemetry(&config.observability);
|
||||
|
||||
let logger = init_global_logger(&config).await;
|
||||
let obs_config = config.observability.clone();
|
||||
tokio::spawn(async move {
|
||||
let result = InitLogStatus::init_start_log(&obs_config).await;
|
||||
match result {
|
||||
Ok(_) => {
|
||||
info!("Logger initialized successfully");
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to initialize logger: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
(logger, guard)
|
||||
}
|
||||
|
||||
/// Get the global logger instance
|
||||
/// This function returns a reference to the global logger instance.
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global logger instance
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::get_logger;
|
||||
///
|
||||
/// let logger = get_logger();
|
||||
/// ```
|
||||
pub fn get_logger() -> &'static Arc<tokio::sync::Mutex<Logger>> {
|
||||
get_global_logger()
|
||||
}
|
||||
|
||||
/// Set the global guard for OpenTelemetry
|
||||
@@ -103,7 +117,7 @@ pub async fn init_obs(endpoint: Option<String>) -> OtelGuard {
|
||||
/// use rustfs_obs::{ init_obs, set_global_guard};
|
||||
///
|
||||
/// async fn init() -> Result<(), Box<dyn std::error::Error>> {
|
||||
/// let guard = init_obs(None).await;
|
||||
/// let (_, guard) = init_obs(None).await;
|
||||
/// set_global_guard(guard)?;
|
||||
/// Ok(())
|
||||
/// }
|
||||
|
||||
@@ -18,7 +18,10 @@
|
||||
//!
|
||||
//! ## feature mark
|
||||
//!
|
||||
//! - `file`: enable file logging enabled by default
|
||||
//! - `gpu`: gpu monitoring function
|
||||
//! - `kafka`: enable kafka metric output
|
||||
//! - `webhook`: enable webhook notifications
|
||||
//! - `full`: includes all functions
|
||||
//!
|
||||
//! to enable gpu monitoring add in cargo toml
|
||||
@@ -38,15 +41,27 @@
|
||||
///
|
||||
/// # #[tokio::main]
|
||||
/// # async fn main() {
|
||||
/// # let guard = init_obs(None).await;
|
||||
/// let (logger, guard) = init_obs(None).await;
|
||||
/// # }
|
||||
/// ```
|
||||
mod config;
|
||||
mod entry;
|
||||
mod global;
|
||||
mod logger;
|
||||
mod metrics;
|
||||
mod sinks;
|
||||
mod system;
|
||||
mod telemetry;
|
||||
mod worker;
|
||||
|
||||
pub use config::AppConfig;
|
||||
pub use config::{AppConfig, LoggerConfig, OtelConfig, SinkConfig};
|
||||
pub use entry::args::Args;
|
||||
pub use entry::audit::{ApiDetails, AuditLogEntry};
|
||||
pub use entry::base::BaseLogEntry;
|
||||
pub use entry::unified::{ConsoleLogEntry, ServerLogEntry, UnifiedLogEntry};
|
||||
pub use entry::{LogKind, LogRecord, ObjectVersion, SerializableLevel};
|
||||
pub use global::*;
|
||||
pub use logger::Logger;
|
||||
pub use logger::{get_global_logger, init_global_logger, start_logger};
|
||||
pub use logger::{log_debug, log_error, log_info, log_trace, log_warn, log_with_context};
|
||||
pub use system::SystemObserver;
|
||||
|
||||
490
crates/obs/src/logger.rs
Normal file
490
crates/obs/src/logger.rs
Normal file
@@ -0,0 +1,490 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::sinks::Sink;
|
||||
use crate::{
|
||||
AppConfig, AuditLogEntry, BaseLogEntry, ConsoleLogEntry, GlobalError, OtelConfig, ServerLogEntry, UnifiedLogEntry, sinks,
|
||||
};
|
||||
use rustfs_config::{APP_NAME, ENVIRONMENT, SERVICE_VERSION};
|
||||
use std::sync::Arc;
|
||||
use std::time::SystemTime;
|
||||
use tokio::sync::mpsc::{self, Receiver, Sender};
|
||||
use tokio::sync::{Mutex, OnceCell};
|
||||
use tracing_core::Level;
|
||||
|
||||
// Add the global instance at the module level
|
||||
static GLOBAL_LOGGER: OnceCell<Arc<Mutex<Logger>>> = OnceCell::const_new();
|
||||
|
||||
/// Server log processor
|
||||
#[derive(Debug)]
|
||||
pub struct Logger {
|
||||
sender: Sender<UnifiedLogEntry>, // Log sending channel
|
||||
queue_capacity: usize,
|
||||
}
|
||||
|
||||
impl Logger {
|
||||
/// Create a new Logger instance
|
||||
/// Returns Logger and corresponding Receiver
|
||||
pub fn new(config: &AppConfig) -> (Self, Receiver<UnifiedLogEntry>) {
|
||||
// Get queue capacity from configuration, or use default values 10000
|
||||
let queue_capacity = config.logger.as_ref().and_then(|l| l.queue_capacity).unwrap_or(10000);
|
||||
let (sender, receiver) = mpsc::channel(queue_capacity);
|
||||
(Logger { sender, queue_capacity }, receiver)
|
||||
}
|
||||
|
||||
/// get the queue capacity
|
||||
/// This function returns the queue capacity.
|
||||
/// # Returns
|
||||
/// The queue capacity
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::Logger;
|
||||
/// async fn example(logger: &Logger) {
|
||||
/// let _ = logger.get_queue_capacity();
|
||||
/// }
|
||||
/// ```
|
||||
pub fn get_queue_capacity(&self) -> usize {
|
||||
self.queue_capacity
|
||||
}
|
||||
|
||||
/// Log a server entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_server"))]
|
||||
pub async fn log_server_entry(&self, entry: ServerLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Server(entry)).await
|
||||
}
|
||||
|
||||
/// Log an audit entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_audit"))]
|
||||
pub async fn log_audit_entry(&self, entry: AuditLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Audit(Box::new(entry))).await
|
||||
}
|
||||
|
||||
/// Log a console entry
|
||||
#[tracing::instrument(skip(self), fields(log_source = "logger_console"))]
|
||||
pub async fn log_console_entry(&self, entry: ConsoleLogEntry) -> Result<(), GlobalError> {
|
||||
self.log_entry(UnifiedLogEntry::Console(entry)).await
|
||||
}
|
||||
|
||||
/// Asynchronous logging of unified log entries
|
||||
#[tracing::instrument(skip_all, fields(log_source = "logger"))]
|
||||
pub async fn log_entry(&self, entry: UnifiedLogEntry) -> Result<(), GlobalError> {
|
||||
// Extract information for tracing based on entry type
|
||||
match &entry {
|
||||
UnifiedLogEntry::Server(server) => {
|
||||
tracing::Span::current()
|
||||
.record("log_level", server.level.0.as_str())
|
||||
.record("log_message", server.base.message.as_deref().unwrap_or("log message not set"))
|
||||
.record("source", &server.source);
|
||||
|
||||
// Generate tracing event based on log level
|
||||
match server.level.0 {
|
||||
Level::ERROR => {
|
||||
tracing::error!(target: "server_logs", message = %server.base.message.as_deref().unwrap_or(""));
|
||||
}
|
||||
Level::WARN => {
|
||||
tracing::warn!(target: "server_logs", message = %server.base.message.as_deref().unwrap_or(""));
|
||||
}
|
||||
Level::INFO => {
|
||||
tracing::info!(target: "server_logs", message = %server.base.message.as_deref().unwrap_or(""));
|
||||
}
|
||||
Level::DEBUG => {
|
||||
tracing::debug!(target: "server_logs", message = %server.base.message.as_deref().unwrap_or(""));
|
||||
}
|
||||
Level::TRACE => {
|
||||
tracing::trace!(target: "server_logs", message = %server.base.message.as_deref().unwrap_or(""));
|
||||
}
|
||||
}
|
||||
}
|
||||
UnifiedLogEntry::Audit(audit) => {
|
||||
tracing::info!(
|
||||
target: "audit_logs",
|
||||
event = %audit.event,
|
||||
api = %audit.api.name.as_deref().unwrap_or("unknown"),
|
||||
message = %audit.base.message.as_deref().unwrap_or("")
|
||||
);
|
||||
}
|
||||
UnifiedLogEntry::Console(console) => {
|
||||
let level_str = match console.level {
|
||||
crate::LogKind::Info => "INFO",
|
||||
crate::LogKind::Warning => "WARN",
|
||||
crate::LogKind::Error => "ERROR",
|
||||
crate::LogKind::Fatal => "FATAL",
|
||||
};
|
||||
|
||||
tracing::info!(
|
||||
target: "console_logs",
|
||||
level = %level_str,
|
||||
node = %console.node_name,
|
||||
message = %console.console_msg
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Send logs to async queue with improved error handling
|
||||
match self.sender.try_send(entry) {
|
||||
Ok(_) => Ok(()),
|
||||
Err(mpsc::error::TrySendError::Full(entry)) => {
|
||||
// Processing strategy when queue is full
|
||||
tracing::warn!("Log queue full, applying backpressure");
|
||||
match tokio::time::timeout(std::time::Duration::from_millis(500), self.sender.send(entry)).await {
|
||||
Ok(Ok(_)) => Ok(()),
|
||||
Ok(Err(_)) => Err(GlobalError::SendFailed("Channel closed")),
|
||||
Err(_) => Err(GlobalError::Timeout("Queue backpressure timeout")),
|
||||
}
|
||||
}
|
||||
Err(mpsc::error::TrySendError::Closed(_)) => Err(GlobalError::SendFailed("Logger channel closed")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Write log with context information
|
||||
/// This function writes log messages with context information.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// - `request_id`: Request ID
|
||||
/// - `user_id`: User ID
|
||||
/// - `fields`: Additional fields
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use tracing_core::Level;
|
||||
/// use rustfs_obs::Logger;
|
||||
///
|
||||
/// async fn example(logger: &Logger) {
|
||||
/// let _ = logger.write_with_context("This is an information message", "example",Level::INFO, Some("req-12345".to_string()), Some("user-6789".to_string()), vec![("endpoint".to_string(), "/api/v1/data".to_string())]).await;
|
||||
/// }
|
||||
pub async fn write_with_context(
|
||||
&self,
|
||||
message: &str,
|
||||
source: &str,
|
||||
level: Level,
|
||||
request_id: Option<String>,
|
||||
user_id: Option<String>,
|
||||
fields: Vec<(String, String)>,
|
||||
) -> Result<(), GlobalError> {
|
||||
let base = BaseLogEntry::new().message(Some(message.to_string())).request_id(request_id);
|
||||
|
||||
let server_entry = ServerLogEntry::new(level, source.to_string())
|
||||
.user_id(user_id)
|
||||
.fields(fields)
|
||||
.with_base(base);
|
||||
|
||||
self.log_server_entry(server_entry).await
|
||||
}
|
||||
|
||||
/// Write log
|
||||
/// This function writes log messages.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// - `level`: Log level
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::Logger;
|
||||
/// use tracing_core::Level;
|
||||
///
|
||||
/// async fn example(logger: &Logger) {
|
||||
/// let _ = logger.write("This is an information message", "example", Level::INFO).await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn write(&self, message: &str, source: &str, level: Level) -> Result<(), GlobalError> {
|
||||
self.write_with_context(message, source, level, None, None, Vec::new()).await
|
||||
}
|
||||
|
||||
/// Shutdown the logger
|
||||
/// This function shuts down the logger.
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::Logger;
|
||||
///
|
||||
/// async fn example(logger: Logger) {
|
||||
/// let _ = logger.shutdown().await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn shutdown(self) -> Result<(), GlobalError> {
|
||||
drop(self.sender); //Close the sending end so that the receiver knows that there is no new message
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Start the log module
|
||||
/// This function starts the log module.
|
||||
/// It initializes the logger and starts the worker to process logs.
|
||||
/// # Parameters
|
||||
/// - `config`: Configuration information
|
||||
/// - `sinks`: A vector of Sink instances
|
||||
/// # Returns
|
||||
/// The global logger instance
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::{AppConfig, start_logger};
|
||||
///
|
||||
/// let config = AppConfig::default();
|
||||
/// let sinks = vec![];
|
||||
/// let logger = start_logger(&config, sinks);
|
||||
/// ```
|
||||
pub fn start_logger(config: &AppConfig, sinks: Vec<Arc<dyn Sink>>) -> Logger {
|
||||
let (logger, receiver) = Logger::new(config);
|
||||
tokio::spawn(crate::worker::start_worker(receiver, sinks));
|
||||
logger
|
||||
}
|
||||
|
||||
/// Initialize the global logger instance
|
||||
/// This function initializes the global logger instance and returns a reference to it.
|
||||
/// If the logger has been initialized before, it will return the existing logger instance.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `config`: Configuration information
|
||||
/// - `sinks`: A vector of Sink instances
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global logger instance
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// use rustfs_obs::{AppConfig,init_global_logger};
|
||||
///
|
||||
/// let config = AppConfig::default();
|
||||
/// let logger = init_global_logger(&config);
|
||||
/// ```
|
||||
pub async fn init_global_logger(config: &AppConfig) -> Arc<Mutex<Logger>> {
|
||||
let sinks = sinks::create_sinks(config).await;
|
||||
let logger = Arc::new(Mutex::new(start_logger(config, sinks)));
|
||||
GLOBAL_LOGGER.set(logger.clone()).expect("Logger already initialized");
|
||||
logger
|
||||
}
|
||||
|
||||
/// Get the global logger instance
|
||||
///
|
||||
/// This function returns a reference to the global logger instance.
|
||||
///
|
||||
/// # Returns
|
||||
/// A reference to the global logger instance
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::get_global_logger;
|
||||
///
|
||||
/// let logger = get_global_logger();
|
||||
/// ```
|
||||
pub fn get_global_logger() -> &'static Arc<Mutex<Logger>> {
|
||||
GLOBAL_LOGGER.get().expect("Logger not initialized")
|
||||
}
|
||||
|
||||
/// Log information
|
||||
/// This function logs information messages.
|
||||
///
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::log_info;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_info("This is an information message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_info(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::INFO).await
|
||||
}
|
||||
|
||||
/// Log error
|
||||
/// This function logs error messages.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::log_error;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_error("This is an error message", "example").await;
|
||||
/// }
|
||||
pub async fn log_error(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::ERROR).await
|
||||
}
|
||||
|
||||
/// Log warning
|
||||
/// This function logs warning messages.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::log_warn;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_warn("This is a warning message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_warn(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::WARN).await
|
||||
}
|
||||
|
||||
/// Log debug
|
||||
/// This function logs debug messages.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::log_debug;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_debug("This is a debug message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_debug(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::DEBUG).await
|
||||
}
|
||||
|
||||
/// Log trace
|
||||
/// This function logs trace messages.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
///
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
///
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use rustfs_obs::log_trace;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_trace("This is a trace message", "example").await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_trace(message: &str, source: &str) -> Result<(), GlobalError> {
|
||||
get_global_logger().lock().await.write(message, source, Level::TRACE).await
|
||||
}
|
||||
|
||||
/// Log with context information
|
||||
/// This function logs messages with context information.
|
||||
/// # Parameters
|
||||
/// - `message`: Message to be logged
|
||||
/// - `source`: Source of the log
|
||||
/// - `level`: Log level
|
||||
/// - `request_id`: Request ID
|
||||
/// - `user_id`: User ID
|
||||
/// - `fields`: Additional fields
|
||||
/// # Returns
|
||||
/// Result indicating whether the operation was successful
|
||||
/// # Example
|
||||
/// ```no_run
|
||||
/// use tracing_core::Level;
|
||||
/// use rustfs_obs::log_with_context;
|
||||
///
|
||||
/// async fn example() {
|
||||
/// let _ = log_with_context("This is an information message", "example", Level::INFO, Some("req-12345".to_string()), Some("user-6789".to_string()), vec![("endpoint".to_string(), "/api/v1/data".to_string())]).await;
|
||||
/// }
|
||||
/// ```
|
||||
pub async fn log_with_context(
|
||||
message: &str,
|
||||
source: &str,
|
||||
level: Level,
|
||||
request_id: Option<String>,
|
||||
user_id: Option<String>,
|
||||
fields: Vec<(String, String)>,
|
||||
) -> Result<(), GlobalError> {
|
||||
get_global_logger()
|
||||
.lock()
|
||||
.await
|
||||
.write_with_context(message, source, level, request_id, user_id, fields)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Log initialization status
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct InitLogStatus {
|
||||
pub timestamp: SystemTime,
|
||||
pub service_name: String,
|
||||
pub version: String,
|
||||
pub environment: String,
|
||||
}
|
||||
|
||||
impl Default for InitLogStatus {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
timestamp: SystemTime::now(),
|
||||
service_name: String::from(APP_NAME),
|
||||
version: SERVICE_VERSION.to_string(),
|
||||
environment: ENVIRONMENT.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl InitLogStatus {
|
||||
pub fn new_config(config: &OtelConfig) -> Self {
|
||||
let config = config.clone();
|
||||
let environment = config.environment.unwrap_or(ENVIRONMENT.to_string());
|
||||
let version = config.service_version.unwrap_or(SERVICE_VERSION.to_string());
|
||||
Self {
|
||||
timestamp: SystemTime::now(),
|
||||
service_name: String::from(APP_NAME),
|
||||
version,
|
||||
environment,
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn init_start_log(config: &OtelConfig) -> Result<(), GlobalError> {
|
||||
let status = Self::new_config(config);
|
||||
log_init_state(Some(status)).await
|
||||
}
|
||||
}
|
||||
|
||||
/// Log initialization details during system startup
|
||||
async fn log_init_state(status: Option<InitLogStatus>) -> Result<(), GlobalError> {
|
||||
let status = status.unwrap_or_default();
|
||||
|
||||
let base_entry = BaseLogEntry::new()
|
||||
.timestamp(chrono::DateTime::from(status.timestamp))
|
||||
.message(Some(format!(
|
||||
"Service initialization started - {} v{} in {}",
|
||||
status.service_name, status.version, status.environment
|
||||
)))
|
||||
.request_id(Some("system_init".to_string()));
|
||||
|
||||
let server_entry = ServerLogEntry::new(Level::INFO, "system_initialization".to_string())
|
||||
.with_base(base_entry)
|
||||
.user_id(Some("system".to_string()));
|
||||
|
||||
get_global_logger().lock().await.log_server_entry(server_entry).await?;
|
||||
Ok(())
|
||||
}
|
||||
178
crates/obs/src/sinks/file.rs
Normal file
178
crates/obs/src/sinks/file.rs
Normal file
@@ -0,0 +1,178 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::sinks::Sink;
|
||||
use crate::{LogRecord, UnifiedLogEntry};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
use tokio::fs::OpenOptions;
|
||||
use tokio::io;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
|
||||
/// File Sink Implementation
|
||||
pub struct FileSink {
|
||||
path: String,
|
||||
buffer_size: usize,
|
||||
writer: Arc<tokio::sync::Mutex<io::BufWriter<tokio::fs::File>>>,
|
||||
entry_count: std::sync::atomic::AtomicUsize,
|
||||
last_flush: std::sync::atomic::AtomicU64,
|
||||
flush_interval_ms: u64, // Time between flushes
|
||||
flush_threshold: usize, // Number of entries before flush
|
||||
}
|
||||
|
||||
impl FileSink {
|
||||
/// Create a new FileSink instance
|
||||
pub async fn new(
|
||||
path: String,
|
||||
buffer_size: usize,
|
||||
flush_interval_ms: u64,
|
||||
flush_threshold: usize,
|
||||
) -> Result<Self, io::Error> {
|
||||
// check if the file exists
|
||||
let file_exists = tokio::fs::metadata(&path).await.is_ok();
|
||||
// if the file not exists, create it
|
||||
if !file_exists {
|
||||
tokio::fs::create_dir_all(std::path::Path::new(&path).parent().unwrap()).await?;
|
||||
tracing::debug!("File does not exist, creating it. Path: {:?}", path)
|
||||
}
|
||||
let file = if file_exists {
|
||||
// If the file exists, open it in append mode
|
||||
tracing::debug!("FileSink: File exists, opening in append mode. Path: {:?}", path);
|
||||
OpenOptions::new().append(true).create(true).open(&path).await?
|
||||
} else {
|
||||
// If the file does not exist, create it
|
||||
tracing::debug!("FileSink: File does not exist, creating a new file.");
|
||||
// Create the file and write a header or initial content if needed
|
||||
OpenOptions::new().create(true).truncate(true).write(true).open(&path).await?
|
||||
};
|
||||
let writer = io::BufWriter::with_capacity(buffer_size, file);
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
Ok(FileSink {
|
||||
path,
|
||||
buffer_size,
|
||||
writer: Arc::new(tokio::sync::Mutex::new(writer)),
|
||||
entry_count: std::sync::atomic::AtomicUsize::new(0),
|
||||
last_flush: std::sync::atomic::AtomicU64::new(now),
|
||||
flush_interval_ms,
|
||||
flush_threshold,
|
||||
})
|
||||
}
|
||||
|
||||
#[allow(dead_code)]
|
||||
async fn initialize_writer(&mut self) -> io::Result<()> {
|
||||
let file = tokio::fs::File::create(&self.path).await?;
|
||||
|
||||
// Use buffer_size to create a buffer writer with a specified capacity
|
||||
let buf_writer = io::BufWriter::with_capacity(self.buffer_size, file);
|
||||
|
||||
// Replace the original writer with the new Mutex
|
||||
self.writer = Arc::new(tokio::sync::Mutex::new(buf_writer));
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Get the current buffer size
|
||||
#[allow(dead_code)]
|
||||
pub fn buffer_size(&self) -> usize {
|
||||
self.buffer_size
|
||||
}
|
||||
|
||||
// How to dynamically adjust the buffer size
|
||||
#[allow(dead_code)]
|
||||
pub async fn set_buffer_size(&mut self, new_size: usize) -> io::Result<()> {
|
||||
if self.buffer_size != new_size {
|
||||
self.buffer_size = new_size;
|
||||
// Reinitialize the writer directly, without checking is_some()
|
||||
self.initialize_writer().await?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
// Check if flushing is needed based on count or time
|
||||
fn should_flush(&self) -> bool {
|
||||
// Check entry count threshold
|
||||
if self.entry_count.load(std::sync::atomic::Ordering::Relaxed) >= self.flush_threshold {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Check time threshold
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
|
||||
let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
|
||||
now - last >= self.flush_interval_ms
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Sink for FileSink {
|
||||
async fn write(&self, entry: &UnifiedLogEntry) {
|
||||
let line = format!("{entry:?}\n");
|
||||
let mut writer = self.writer.lock().await;
|
||||
|
||||
if let Err(e) = writer.write_all(line.as_bytes()).await {
|
||||
eprintln!(
|
||||
"Failed to write log to file {}: {},entry timestamp:{:?}",
|
||||
self.path,
|
||||
e,
|
||||
entry.get_timestamp()
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
// Only flush periodically to improve performance
|
||||
// Logic to determine when to flush could be added here
|
||||
// Increment the entry count
|
||||
self.entry_count.fetch_add(1, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
// Check if we should flush
|
||||
if self.should_flush() {
|
||||
if let Err(e) = writer.flush().await {
|
||||
eprintln!("Failed to flush log file {}: {}", self.path, e);
|
||||
return;
|
||||
}
|
||||
|
||||
// Reset counters
|
||||
self.entry_count.store(0, std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
|
||||
self.last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FileSink {
|
||||
fn drop(&mut self) {
|
||||
let writer = self.writer.clone();
|
||||
let path = self.path.clone();
|
||||
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
rt.block_on(async {
|
||||
let mut writer = writer.lock().await;
|
||||
if let Err(e) = writer.flush().await {
|
||||
eprintln!("Failed to flush log file {path}: {e}");
|
||||
}
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
179
crates/obs/src/sinks/kafka.rs
Normal file
179
crates/obs/src/sinks/kafka.rs
Normal file
@@ -0,0 +1,179 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::sinks::Sink;
|
||||
use crate::{LogRecord, UnifiedLogEntry};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Kafka Sink Implementation
|
||||
pub struct KafkaSink {
|
||||
producer: rdkafka::producer::FutureProducer,
|
||||
topic: String,
|
||||
batch_size: usize,
|
||||
batch_timeout_ms: u64,
|
||||
entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
|
||||
last_flush: Arc<std::sync::atomic::AtomicU64>,
|
||||
}
|
||||
|
||||
impl KafkaSink {
|
||||
/// Create a new KafkaSink instance
|
||||
pub fn new(producer: rdkafka::producer::FutureProducer, topic: String, batch_size: usize, batch_timeout_ms: u64) -> Self {
|
||||
// Create Arc-wrapped values first
|
||||
let entries = Arc::new(tokio::sync::Mutex::new(Vec::with_capacity(batch_size)));
|
||||
let last_flush = Arc::new(std::sync::atomic::AtomicU64::new(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64,
|
||||
));
|
||||
let sink = KafkaSink {
|
||||
producer: producer.clone(),
|
||||
topic: topic.clone(),
|
||||
batch_size,
|
||||
batch_timeout_ms,
|
||||
entries: entries.clone(),
|
||||
last_flush: last_flush.clone(),
|
||||
};
|
||||
|
||||
// Start background flusher
|
||||
tokio::spawn(Self::periodic_flush(producer, topic, entries, last_flush, batch_timeout_ms));
|
||||
|
||||
sink
|
||||
}
|
||||
|
||||
/// Add a getter method to read the batch_timeout_ms field
|
||||
#[allow(dead_code)]
|
||||
pub fn batch_timeout(&self) -> u64 {
|
||||
self.batch_timeout_ms
|
||||
}
|
||||
|
||||
/// Add a method to dynamically adjust the timeout if needed
|
||||
#[allow(dead_code)]
|
||||
pub fn set_batch_timeout(&mut self, new_timeout_ms: u64) {
|
||||
self.batch_timeout_ms = new_timeout_ms;
|
||||
}
|
||||
|
||||
async fn periodic_flush(
|
||||
producer: rdkafka::producer::FutureProducer,
|
||||
topic: String,
|
||||
entries: Arc<tokio::sync::Mutex<Vec<UnifiedLogEntry>>>,
|
||||
last_flush: Arc<std::sync::atomic::AtomicU64>,
|
||||
timeout_ms: u64,
|
||||
) {
|
||||
loop {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(timeout_ms / 2)).await;
|
||||
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
|
||||
let last = last_flush.load(std::sync::atomic::Ordering::Relaxed);
|
||||
|
||||
if now - last >= timeout_ms {
|
||||
let mut batch = entries.lock().await;
|
||||
if !batch.is_empty() {
|
||||
Self::send_batch(&producer, &topic, batch.drain(..).collect()).await;
|
||||
last_flush.store(now, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async fn send_batch(producer: &rdkafka::producer::FutureProducer, topic: &str, entries: Vec<UnifiedLogEntry>) {
|
||||
for entry in entries {
|
||||
let payload = match serde_json::to_string(&entry) {
|
||||
Ok(p) => p,
|
||||
Err(e) => {
|
||||
eprintln!("Failed to serialize log entry: {e}");
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let span_id = entry.get_timestamp().to_rfc3339();
|
||||
|
||||
let _ = producer
|
||||
.send(
|
||||
rdkafka::producer::FutureRecord::to(topic).payload(&payload).key(&span_id),
|
||||
std::time::Duration::from_secs(5),
|
||||
)
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Sink for KafkaSink {
|
||||
async fn write(&self, entry: &UnifiedLogEntry) {
|
||||
let mut batch = self.entries.lock().await;
|
||||
batch.push(entry.clone());
|
||||
|
||||
let should_flush_by_size = batch.len() >= self.batch_size;
|
||||
let should_flush_by_time = {
|
||||
let now = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64;
|
||||
let last = self.last_flush.load(std::sync::atomic::Ordering::Relaxed);
|
||||
now - last >= self.batch_timeout_ms
|
||||
};
|
||||
|
||||
if should_flush_by_size || should_flush_by_time {
|
||||
// Existing flush logic
|
||||
let entries_to_send: Vec<UnifiedLogEntry> = batch.drain(..).collect();
|
||||
let producer = self.producer.clone();
|
||||
let topic = self.topic.clone();
|
||||
|
||||
self.last_flush.store(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64,
|
||||
std::sync::atomic::Ordering::Relaxed,
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
KafkaSink::send_batch(&producer, &topic, entries_to_send).await;
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for KafkaSink {
|
||||
fn drop(&mut self) {
|
||||
// Perform any necessary cleanup here
|
||||
// For example, you might want to flush any remaining entries
|
||||
let producer = self.producer.clone();
|
||||
let topic = self.topic.clone();
|
||||
let entries = self.entries.clone();
|
||||
let last_flush = self.last_flush.clone();
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut batch = entries.lock().await;
|
||||
if !batch.is_empty() {
|
||||
KafkaSink::send_batch(&producer, &topic, batch.drain(..).collect()).await;
|
||||
last_flush.store(
|
||||
std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_millis() as u64,
|
||||
std::sync::atomic::Ordering::Relaxed,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
eprintln!("Dropping KafkaSink with topic: {0}", self.topic);
|
||||
}
|
||||
}
|
||||
123
crates/obs/src/sinks/mod.rs
Normal file
123
crates/obs/src/sinks/mod.rs
Normal file
@@ -0,0 +1,123 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::{AppConfig, SinkConfig, UnifiedLogEntry};
|
||||
use async_trait::async_trait;
|
||||
use std::sync::Arc;
|
||||
|
||||
#[cfg(feature = "file")]
|
||||
mod file;
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
mod kafka;
|
||||
#[cfg(feature = "webhook")]
|
||||
mod webhook;
|
||||
|
||||
/// Sink Trait definition, asynchronously write logs
|
||||
#[async_trait]
|
||||
pub trait Sink: Send + Sync {
|
||||
async fn write(&self, entry: &UnifiedLogEntry);
|
||||
}
|
||||
|
||||
/// Create a list of Sink instances
|
||||
pub async fn create_sinks(config: &AppConfig) -> Vec<Arc<dyn Sink>> {
|
||||
let mut sinks: Vec<Arc<dyn Sink>> = Vec::new();
|
||||
|
||||
for sink_config in &config.sinks {
|
||||
match sink_config {
|
||||
#[cfg(all(feature = "kafka", target_os = "linux"))]
|
||||
SinkConfig::Kafka(kafka_config) => {
|
||||
match rdkafka::config::ClientConfig::new()
|
||||
.set("bootstrap.servers", &kafka_config.brokers)
|
||||
.set("message.timeout.ms", "5000")
|
||||
.create()
|
||||
{
|
||||
Ok(producer) => {
|
||||
sinks.push(Arc::new(kafka::KafkaSink::new(
|
||||
producer,
|
||||
kafka_config.topic.clone(),
|
||||
kafka_config
|
||||
.batch_size
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_KAFKA_BATCH_SIZE),
|
||||
kafka_config
|
||||
.batch_timeout_ms
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_KAFKA_BATCH_TIMEOUT_MS),
|
||||
)));
|
||||
tracing::info!("Kafka sink created for topic: {}", kafka_config.topic);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to create Kafka producer: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(feature = "webhook")]
|
||||
SinkConfig::Webhook(webhook_config) => {
|
||||
sinks.push(Arc::new(webhook::WebhookSink::new(
|
||||
webhook_config.endpoint.clone(),
|
||||
webhook_config.auth_token.clone(),
|
||||
webhook_config
|
||||
.max_retries
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_WEBHOOK_MAX_RETRIES),
|
||||
webhook_config
|
||||
.retry_delay_ms
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_WEBHOOK_RETRY_DELAY_MS),
|
||||
)));
|
||||
tracing::info!("Webhook sink created for endpoint: {}", webhook_config.endpoint);
|
||||
}
|
||||
#[cfg(feature = "file")]
|
||||
SinkConfig::File(file_config) => {
|
||||
tracing::debug!("FileSink: Using path: {}", file_config.path);
|
||||
match file::FileSink::new(
|
||||
std::path::Path::new(&file_config.path)
|
||||
.join(rustfs_config::DEFAULT_SINK_FILE_LOG_FILE)
|
||||
.to_string_lossy()
|
||||
.to_string(),
|
||||
file_config
|
||||
.buffer_size
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_FILE_BUFFER_SIZE),
|
||||
file_config
|
||||
.flush_interval_ms
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_FILE_FLUSH_INTERVAL_MS),
|
||||
file_config
|
||||
.flush_threshold
|
||||
.unwrap_or(rustfs_config::observability::DEFAULT_SINKS_FILE_FLUSH_THRESHOLD),
|
||||
)
|
||||
.await
|
||||
{
|
||||
Ok(sink) => {
|
||||
sinks.push(Arc::new(sink));
|
||||
tracing::info!("File sink created for path: {}", file_config.path);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::error!("Failed to create File sink: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
#[cfg(any(not(feature = "kafka"), not(target_os = "linux")))]
|
||||
SinkConfig::Kafka(_) => {
|
||||
tracing::warn!("Kafka sink is configured but the 'kafka' feature is not enabled");
|
||||
}
|
||||
#[cfg(not(feature = "webhook"))]
|
||||
SinkConfig::Webhook(_) => {
|
||||
tracing::warn!("Webhook sink is configured but the 'webhook' feature is not enabled");
|
||||
}
|
||||
#[cfg(not(feature = "file"))]
|
||||
SinkConfig::File(_) => {
|
||||
tracing::warn!("File sink is configured but the 'file' feature is not enabled");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
sinks
|
||||
}
|
||||
84
crates/obs/src/sinks/webhook.rs
Normal file
84
crates/obs/src/sinks/webhook.rs
Normal file
@@ -0,0 +1,84 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::UnifiedLogEntry;
|
||||
use crate::sinks::Sink;
|
||||
use async_trait::async_trait;
|
||||
|
||||
/// Webhook Sink Implementation
|
||||
pub struct WebhookSink {
|
||||
endpoint: String,
|
||||
auth_token: String,
|
||||
client: reqwest::Client,
|
||||
max_retries: usize,
|
||||
retry_delay_ms: u64,
|
||||
}
|
||||
|
||||
impl WebhookSink {
|
||||
pub fn new(endpoint: String, auth_token: String, max_retries: usize, retry_delay_ms: u64) -> Self {
|
||||
WebhookSink {
|
||||
endpoint,
|
||||
auth_token,
|
||||
client: reqwest::Client::builder()
|
||||
.timeout(std::time::Duration::from_secs(10))
|
||||
.build()
|
||||
.unwrap_or_else(|_| reqwest::Client::new()),
|
||||
max_retries,
|
||||
retry_delay_ms,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait]
|
||||
impl Sink for WebhookSink {
|
||||
async fn write(&self, entry: &UnifiedLogEntry) {
|
||||
let mut retries = 0;
|
||||
let url = self.endpoint.clone();
|
||||
let entry_clone = entry.clone();
|
||||
let auth_value = reqwest::header::HeaderValue::from_str(format!("Bearer {}", self.auth_token.clone()).as_str()).unwrap();
|
||||
while retries < self.max_retries {
|
||||
match self
|
||||
.client
|
||||
.post(&url)
|
||||
.header(reqwest::header::AUTHORIZATION, auth_value.clone())
|
||||
.json(&entry_clone)
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(response) if response.status().is_success() => {
|
||||
return;
|
||||
}
|
||||
_ => {
|
||||
retries += 1;
|
||||
if retries < self.max_retries {
|
||||
tokio::time::sleep(tokio::time::Duration::from_millis(
|
||||
self.retry_delay_ms * (1 << retries), // Exponential backoff
|
||||
))
|
||||
.await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
eprintln!("Failed to send log to webhook after {0} retries", self.max_retries);
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for WebhookSink {
|
||||
fn drop(&mut self) {
|
||||
// Perform any necessary cleanup here
|
||||
// For example, you might want to log that the sink is being dropped
|
||||
eprintln!("Dropping WebhookSink with URL: {0}", self.endpoint);
|
||||
}
|
||||
}
|
||||
@@ -12,8 +12,7 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::{IS_OBSERVABILITY_ENABLED, OBSERVABILITY_METER_NAME};
|
||||
use crate::OtelConfig;
|
||||
use flexi_logger::{
|
||||
Age, Cleanup, Criterion, DeferredNow, FileSpec, LogSpecification, Naming, Record, WriteMode,
|
||||
WriteMode::{AsyncWith, BufferAndFlush},
|
||||
@@ -64,8 +63,7 @@ use tracing_subscriber::{EnvFilter, Layer, layer::SubscriberExt, util::Subscribe
|
||||
/// - The tracer provider (for distributed tracing)
|
||||
/// - The meter provider (for metrics collection)
|
||||
/// - The logger provider (for structured logging)
|
||||
///
|
||||
/// Implement Debug trait correctly, rather than using derive, as some fields may not have implemented Debug
|
||||
// Implement Debug trait correctly, rather than using derive, as some fields may not have implemented Debug
|
||||
pub struct OtelGuard {
|
||||
tracer_provider: Option<SdkTracerProvider>,
|
||||
meter_provider: Option<SdkMeterProvider>,
|
||||
@@ -303,8 +301,6 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> OtelGuard {
|
||||
logger_level,
|
||||
env::var("RUST_LOG").unwrap_or_else(|_| "Not set".to_string())
|
||||
);
|
||||
IS_OBSERVABILITY_ENABLED.set(true).ok();
|
||||
OBSERVABILITY_METER_NAME.set(service_name.to_string()).ok();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -12,10 +12,16 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//opa env vars
|
||||
pub const ENV_POLICY_PLUGIN_OPA_URL: &str = "RUSTFS_POLICY_PLUGIN_URL";
|
||||
pub const ENV_POLICY_PLUGIN_AUTH_TOKEN: &str = "RUSTFS_POLICY_PLUGIN_AUTH_TOKEN";
|
||||
use crate::{UnifiedLogEntry, sinks::Sink};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::mpsc::Receiver;
|
||||
|
||||
pub const ENV_POLICY_PLUGIN_KEYS: &[&str] = &[ENV_POLICY_PLUGIN_OPA_URL, ENV_POLICY_PLUGIN_AUTH_TOKEN];
|
||||
|
||||
pub const POLICY_PLUGIN_SUB_SYS: &str = "policy_plugin";
|
||||
/// Start the log processing worker thread
|
||||
pub(crate) async fn start_worker(receiver: Receiver<UnifiedLogEntry>, sinks: Vec<Arc<dyn Sink>>) {
|
||||
let mut receiver = receiver;
|
||||
while let Some(entry) = receiver.recv().await {
|
||||
for sink in &sinks {
|
||||
sink.write(&entry).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -29,8 +29,7 @@ documentation = "https://docs.rs/rustfs-policy/latest/rustfs_policy/"
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
rustfs-config = { workspace = true, features = ["constants","opa"] }
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
tokio.workspace = true
|
||||
time = { workspace = true, features = ["serde-human-readable"] }
|
||||
serde = { workspace = true, features = ["derive", "rc"] }
|
||||
serde_json.workspace = true
|
||||
@@ -42,10 +41,6 @@ rand.workspace = true
|
||||
base64-simd = { workspace = true }
|
||||
jsonwebtoken = { workspace = true }
|
||||
regex = { workspace = true }
|
||||
reqwest.workspace = true
|
||||
chrono.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
test-case.workspace = true
|
||||
temp-env = { workspace = true }
|
||||
|
||||
@@ -17,7 +17,6 @@ mod doc;
|
||||
mod effect;
|
||||
mod function;
|
||||
mod id;
|
||||
pub mod opa;
|
||||
#[allow(clippy::module_inception)]
|
||||
mod policy;
|
||||
mod principal;
|
||||
|
||||
@@ -1,288 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use crate::policy::Args as PArgs;
|
||||
use rustfs_config::{ENV_PREFIX, opa::*};
|
||||
use serde::Deserialize;
|
||||
use serde_json::json;
|
||||
use std::{collections::HashMap, env, time::Duration};
|
||||
use tracing::{error, info};
|
||||
|
||||
#[derive(Debug, Clone, Default)]
|
||||
pub struct Args {
|
||||
pub url: String,
|
||||
pub auth_token: String,
|
||||
}
|
||||
impl Args {
|
||||
pub fn enable(&self) -> bool {
|
||||
!self.url.is_empty()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct AuthZPlugin {
|
||||
client: reqwest::Client,
|
||||
args: Args,
|
||||
}
|
||||
|
||||
fn check() -> Result<(), String> {
|
||||
let env_list = env::vars();
|
||||
let mut candidate = HashMap::new();
|
||||
let prefix = format!("{ENV_PREFIX}{POLICY_PLUGIN_SUB_SYS}").to_uppercase();
|
||||
for (key, value) in env_list {
|
||||
if key.starts_with(&prefix) {
|
||||
candidate.insert(key.to_string(), value);
|
||||
}
|
||||
}
|
||||
|
||||
//check required env vars
|
||||
if candidate.remove(ENV_POLICY_PLUGIN_OPA_URL).is_none() {
|
||||
return Err(format!("Missing required env var: {ENV_POLICY_PLUGIN_OPA_URL}"));
|
||||
}
|
||||
|
||||
// check optional env vars
|
||||
candidate.remove(ENV_POLICY_PLUGIN_AUTH_TOKEN);
|
||||
if !candidate.is_empty() {
|
||||
return Err(format!("Invalid env vars: {candidate:?}"));
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
async fn validate(config: &Args) -> Result<(), String> {
|
||||
let client = reqwest::Client::new();
|
||||
|
||||
match client.post(&config.url).send().await {
|
||||
Ok(resp) => {
|
||||
match resp.status() {
|
||||
reqwest::StatusCode::OK => {
|
||||
info!("OPA is ready to accept requests.");
|
||||
}
|
||||
_ => {
|
||||
return Err(format!("OPA returned an error: {}", resp.status()));
|
||||
}
|
||||
};
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(format!("Error connecting to OPA: {err}"));
|
||||
}
|
||||
};
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub async fn lookup_config() -> Result<Args, String> {
|
||||
let args = Args::default();
|
||||
|
||||
let get_cfg =
|
||||
|cfg: &str| -> Result<String, String> { env::var(cfg).map_err(|e| format!("Error getting env var {cfg}: {e:?}")) };
|
||||
|
||||
let url = get_cfg(ENV_POLICY_PLUGIN_OPA_URL);
|
||||
if url.is_err() {
|
||||
info!("OPA is not enabled.");
|
||||
return Ok(args);
|
||||
}
|
||||
check()?;
|
||||
let args = Args {
|
||||
url: url.ok().unwrap(),
|
||||
auth_token: get_cfg(ENV_POLICY_PLUGIN_AUTH_TOKEN).unwrap_or_default(),
|
||||
};
|
||||
validate(&args).await?;
|
||||
Ok(args)
|
||||
}
|
||||
|
||||
impl AuthZPlugin {
|
||||
pub fn new(config: Args) -> Self {
|
||||
let client = reqwest::Client::builder()
|
||||
.timeout(Duration::from_secs(5))
|
||||
.connect_timeout(Duration::from_secs(1))
|
||||
.pool_max_idle_per_host(10)
|
||||
.pool_idle_timeout(Some(Duration::from_secs(60)))
|
||||
.tcp_keepalive(Some(Duration::from_secs(30)))
|
||||
.tcp_nodelay(true)
|
||||
.http2_keep_alive_interval(Some(Duration::from_secs(30)))
|
||||
.http2_keep_alive_timeout(Duration::from_secs(15))
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
Self { client, args: config }
|
||||
}
|
||||
|
||||
pub async fn is_allowed(&self, args: &PArgs<'_>) -> bool {
|
||||
let payload = self.build_opa_input(args);
|
||||
|
||||
let mut request = self.client.post(self.args.url.clone()).json(&payload);
|
||||
if !self.args.auth_token.is_empty() {
|
||||
request = request.header("Authorization", format!("Bearer {}", self.args.auth_token));
|
||||
}
|
||||
|
||||
match request.send().await {
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
if !status.is_success() {
|
||||
error!("OPA returned non-success status: {}", status);
|
||||
return false;
|
||||
}
|
||||
|
||||
match resp.json::<OpaResponseEnum>().await {
|
||||
Ok(response_enum) => match response_enum {
|
||||
OpaResponseEnum::SimpleResult(result) => result.result,
|
||||
OpaResponseEnum::AllowResult(response) => response.result.allow,
|
||||
},
|
||||
Err(err) => {
|
||||
error!("Error parsing OPA response: {:?}", err);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
error!("Error sending request to OPA: {:?}", err);
|
||||
false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn build_opa_input(&self, args: &PArgs<'_>) -> serde_json::Value {
|
||||
let groups = match args.groups {
|
||||
Some(g) => g.clone(),
|
||||
None => vec![],
|
||||
};
|
||||
let action_str: &str = (&args.action).into();
|
||||
json!({
|
||||
// Core authorization parameters for OPA policy evaluation
|
||||
"input":{
|
||||
"identity": {
|
||||
"account": args.account,
|
||||
"groups": groups,
|
||||
"is_owner": args.is_owner,
|
||||
"claims": args.claims
|
||||
},
|
||||
|
||||
"resource": {
|
||||
"bucket": args.bucket,
|
||||
"object": args.object,
|
||||
"arn": if args.object.is_empty() {
|
||||
format!("arn:aws:s3:::{}", args.bucket)
|
||||
} else {
|
||||
format!("arn:aws:s3:::{}/{}", args.bucket, args.object)
|
||||
}
|
||||
},
|
||||
|
||||
"action": action_str,
|
||||
|
||||
"context": {
|
||||
"conditions": args.conditions,
|
||||
"deny_only": args.deny_only,
|
||||
"timestamp": chrono::Utc::now().to_rfc3339()
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct OpaResultAllow {
|
||||
allow: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct OpaResult {
|
||||
result: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize, Default)]
|
||||
struct OpaResponse {
|
||||
result: OpaResultAllow,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(untagged)]
|
||||
enum OpaResponseEnum {
|
||||
SimpleResult(OpaResult),
|
||||
AllowResult(OpaResponse),
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use temp_env;
|
||||
|
||||
#[test]
|
||||
fn test_check_valid_config() {
|
||||
// Use temp_env to temporarily set environment variables
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", Some("http://localhost:8181/v1/data/rustfs/authz/allow")),
|
||||
("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", Some("test-token")),
|
||||
],
|
||||
|| {
|
||||
assert!(check().is_ok());
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_missing_required_env() {
|
||||
temp_env::with_var_unset("RUSTFS_POLICY_PLUGIN_URL", || {
|
||||
temp_env::with_var("RUSTFS_POLICY_PLUGIN_AUTH_TOKEN", Some("test-token"), || {
|
||||
let result = check();
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains("Missing required env var"));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_invalid_env_vars() {
|
||||
temp_env::with_vars(
|
||||
[
|
||||
("RUSTFS_POLICY_PLUGIN_URL", Some("http://localhost:8181/v1/data/rustfs/authz/allow")),
|
||||
("RUSTFS_POLICY_PLUGIN_INVALID", Some("invalid-value")),
|
||||
],
|
||||
|| {
|
||||
let result = check();
|
||||
assert!(result.is_err());
|
||||
assert!(result.unwrap_err().contains("Invalid env vars"));
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_lookup_config_not_enabled() {
|
||||
temp_env::with_var_unset("RUSTFS_POLICY_PLUGIN_URL", || {
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
let result = rt.block_on(async { lookup_config().await });
|
||||
|
||||
// Should return the default empty Args
|
||||
assert!(result.is_ok());
|
||||
let args = result.unwrap();
|
||||
assert!(!args.enable());
|
||||
assert_eq!(args.url, "");
|
||||
assert_eq!(args.auth_token, "");
|
||||
});
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_args_enable() {
|
||||
// Test Args enable method
|
||||
let args_enabled = Args {
|
||||
url: "http://localhost:8181".to_string(),
|
||||
auth_token: "token".to_string(),
|
||||
};
|
||||
assert!(args_enabled.enable());
|
||||
|
||||
let args_disabled = Args {
|
||||
url: "".to_string(),
|
||||
auth_token: "".to_string(),
|
||||
};
|
||||
assert!(!args_disabled.enable());
|
||||
}
|
||||
}
|
||||
@@ -17,6 +17,7 @@ use crate::error::{Error, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use serde_json::Value;
|
||||
use std::collections::{HashMap, HashSet};
|
||||
|
||||
/// DEFAULT_VERSION is the default version.
|
||||
/// https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_policies_elements_version.html
|
||||
pub const DEFAULT_VERSION: &str = "2012-10-17";
|
||||
@@ -154,8 +155,8 @@ impl Validator for Policy {
|
||||
type Error = Error;
|
||||
|
||||
fn is_valid(&self) -> Result<()> {
|
||||
if !self.version.is_empty() && !self.version.eq(DEFAULT_VERSION) {
|
||||
return Err(IamError::InvalidVersion(self.version.clone()).into());
|
||||
if !self.id.is_empty() && !self.id.eq(DEFAULT_VERSION) {
|
||||
return Err(IamError::InvalidVersion(self.id.0.clone()).into());
|
||||
}
|
||||
|
||||
for statement in self.statements.iter() {
|
||||
@@ -213,8 +214,8 @@ impl Validator for BucketPolicy {
|
||||
type Error = Error;
|
||||
|
||||
fn is_valid(&self) -> Result<()> {
|
||||
if !self.version.is_empty() && !self.version.eq(DEFAULT_VERSION) {
|
||||
return Err(IamError::InvalidVersion(self.version.clone()).into());
|
||||
if !self.id.is_empty() && !self.id.eq(DEFAULT_VERSION) {
|
||||
return Err(IamError::InvalidVersion(self.id.0.clone()).into());
|
||||
}
|
||||
|
||||
for statement in self.statements.iter() {
|
||||
|
||||
@@ -44,15 +44,6 @@ rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] }
|
||||
serde_json.workspace = true
|
||||
md-5 = { workspace = true }
|
||||
tracing.workspace = true
|
||||
thiserror.workspace = true
|
||||
base64.workspace = true
|
||||
sha1.workspace = true
|
||||
sha2.workspace = true
|
||||
base64-simd.workspace = true
|
||||
crc64fast-nvme.workspace = true
|
||||
s3s.workspace = true
|
||||
hex-simd.workspace = true
|
||||
crc32c.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
tokio-test = { workspace = true }
|
||||
tokio-test = { workspace = true }
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,73 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use thiserror::Error;
|
||||
|
||||
/// SHA256 mismatch error - when content SHA256 does not match what was sent from client
|
||||
#[derive(Error, Debug, Clone, PartialEq)]
|
||||
#[error("Bad sha256: Expected {expected_sha256} does not match calculated {calculated_sha256}")]
|
||||
pub struct Sha256Mismatch {
|
||||
pub expected_sha256: String,
|
||||
pub calculated_sha256: String,
|
||||
}
|
||||
|
||||
/// Bad digest error - Content-MD5 you specified did not match what we received
|
||||
#[derive(Error, Debug, Clone, PartialEq)]
|
||||
#[error("Bad digest: Expected {expected_md5} does not match calculated {calculated_md5}")]
|
||||
pub struct BadDigest {
|
||||
pub expected_md5: String,
|
||||
pub calculated_md5: String,
|
||||
}
|
||||
|
||||
/// Size too small error - reader size too small
|
||||
#[derive(Error, Debug, Clone, PartialEq)]
|
||||
#[error("Size small: got {got}, want {want}")]
|
||||
pub struct SizeTooSmall {
|
||||
pub want: i64,
|
||||
pub got: i64,
|
||||
}
|
||||
|
||||
/// Size too large error - reader size too large
|
||||
#[derive(Error, Debug, Clone, PartialEq)]
|
||||
#[error("Size large: got {got}, want {want}")]
|
||||
pub struct SizeTooLarge {
|
||||
pub want: i64,
|
||||
pub got: i64,
|
||||
}
|
||||
|
||||
/// Size mismatch error
|
||||
#[derive(Error, Debug, Clone, PartialEq)]
|
||||
#[error("Size mismatch: got {got}, want {want}")]
|
||||
pub struct SizeMismatch {
|
||||
pub want: i64,
|
||||
pub got: i64,
|
||||
}
|
||||
|
||||
/// Checksum mismatch error - when content checksum does not match what was sent from client
|
||||
#[derive(Error, Debug, Clone, PartialEq)]
|
||||
#[error("Bad checksum: Want {want} does not match calculated {got}")]
|
||||
pub struct ChecksumMismatch {
|
||||
pub want: String,
|
||||
pub got: String,
|
||||
}
|
||||
|
||||
/// Invalid checksum error
|
||||
#[derive(Error, Debug, Clone, PartialEq)]
|
||||
#[error("invalid checksum")]
|
||||
pub struct InvalidChecksum;
|
||||
|
||||
/// Check if an error is a checksum mismatch
|
||||
pub fn is_checksum_mismatch(err: &(dyn std::error::Error + 'static)) -> bool {
|
||||
err.downcast_ref::<ChecksumMismatch>().is_some()
|
||||
}
|
||||
@@ -51,7 +51,6 @@ mod tests {
|
||||
|
||||
use crate::{CompressReader, EncryptReader, EtagReader, HashReader};
|
||||
use crate::{WarpReader, resolve_etag_generic};
|
||||
use md5::Md5;
|
||||
use rustfs_utils::compress::CompressionAlgorithm;
|
||||
use std::io::Cursor;
|
||||
use tokio::io::BufReader;
|
||||
@@ -73,7 +72,7 @@ mod tests {
|
||||
let reader = BufReader::new(Cursor::new(&data[..]));
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
let mut hash_reader =
|
||||
HashReader::new(reader, data.len() as i64, data.len() as i64, Some("hash_etag".to_string()), None, false).unwrap();
|
||||
HashReader::new(reader, data.len() as i64, data.len() as i64, Some("hash_etag".to_string()), false).unwrap();
|
||||
|
||||
// Test HashReader ETag resolution
|
||||
assert_eq!(resolve_etag_generic(&mut hash_reader), Some("hash_etag".to_string()));
|
||||
@@ -106,30 +105,20 @@ mod tests {
|
||||
assert_eq!(resolve_etag_generic(&mut encrypt_reader), Some("encrypt_etag".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_complex_nesting() {
|
||||
use md5::Digest;
|
||||
use tokio::io::AsyncReadExt;
|
||||
#[test]
|
||||
fn test_complex_nesting() {
|
||||
let data = b"test data for complex nesting";
|
||||
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(data);
|
||||
let etag = hasher.finalize();
|
||||
let etag_hex = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
|
||||
let reader = BufReader::new(Cursor::new(&data[..]));
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
// Create a complex nested structure: CompressReader<EncryptReader<EtagReader<BufReader<Cursor>>>>
|
||||
let etag_reader = EtagReader::new(reader, Some(etag_hex.clone()));
|
||||
let etag_reader = EtagReader::new(reader, Some("nested_etag".to_string()));
|
||||
let key = [0u8; 32];
|
||||
let nonce = [0u8; 12];
|
||||
let encrypt_reader = EncryptReader::new(etag_reader, key, nonce);
|
||||
let mut compress_reader = CompressReader::new(encrypt_reader, CompressionAlgorithm::Gzip);
|
||||
|
||||
compress_reader.read_to_end(&mut Vec::new()).await.unwrap();
|
||||
|
||||
// Test that nested structure can resolve ETag
|
||||
assert_eq!(resolve_etag_generic(&mut compress_reader), Some(etag_hex));
|
||||
assert_eq!(resolve_etag_generic(&mut compress_reader), Some("nested_etag".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -138,80 +127,51 @@ mod tests {
|
||||
let reader = BufReader::new(Cursor::new(&data[..]));
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
// Create nested structure: CompressReader<HashReader<BufReader<Cursor>>>
|
||||
let hash_reader = HashReader::new(
|
||||
reader,
|
||||
data.len() as i64,
|
||||
data.len() as i64,
|
||||
Some("hash_nested_etag".to_string()),
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
let hash_reader =
|
||||
HashReader::new(reader, data.len() as i64, data.len() as i64, Some("hash_nested_etag".to_string()), false).unwrap();
|
||||
let mut compress_reader = CompressReader::new(hash_reader, CompressionAlgorithm::Deflate);
|
||||
|
||||
// Test that nested HashReader can be resolved
|
||||
assert_eq!(resolve_etag_generic(&mut compress_reader), Some("hash_nested_etag".to_string()));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_comprehensive_etag_extraction() {
|
||||
use md5::Digest;
|
||||
use tokio::io::AsyncReadExt;
|
||||
#[test]
|
||||
fn test_comprehensive_etag_extraction() {
|
||||
println!("🔍 Testing comprehensive ETag extraction with real reader types...");
|
||||
|
||||
// Test 1: Simple EtagReader
|
||||
let data1 = b"simple test";
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(data1);
|
||||
let etag = hasher.finalize();
|
||||
let etag_hex = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
let reader1 = BufReader::new(Cursor::new(&data1[..]));
|
||||
let reader1 = Box::new(WarpReader::new(reader1));
|
||||
let mut etag_reader = EtagReader::new(reader1, Some(etag_hex.clone()));
|
||||
etag_reader.read_to_end(&mut Vec::new()).await.unwrap();
|
||||
assert_eq!(resolve_etag_generic(&mut etag_reader), Some(etag_hex.clone()));
|
||||
let mut etag_reader = EtagReader::new(reader1, Some("simple_etag".to_string()));
|
||||
assert_eq!(resolve_etag_generic(&mut etag_reader), Some("simple_etag".to_string()));
|
||||
|
||||
// Test 2: HashReader with ETag
|
||||
let data2 = b"hash test";
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(data2);
|
||||
let etag = hasher.finalize();
|
||||
let etag_hex = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
let reader2 = BufReader::new(Cursor::new(&data2[..]));
|
||||
let reader2 = Box::new(WarpReader::new(reader2));
|
||||
let mut hash_reader =
|
||||
HashReader::new(reader2, data2.len() as i64, data2.len() as i64, Some(etag_hex.clone()), None, false).unwrap();
|
||||
hash_reader.read_to_end(&mut Vec::new()).await.unwrap();
|
||||
assert_eq!(resolve_etag_generic(&mut hash_reader), Some(etag_hex.clone()));
|
||||
HashReader::new(reader2, data2.len() as i64, data2.len() as i64, Some("hash_etag".to_string()), false).unwrap();
|
||||
assert_eq!(resolve_etag_generic(&mut hash_reader), Some("hash_etag".to_string()));
|
||||
|
||||
// Test 3: Single wrapper - CompressReader<EtagReader>
|
||||
let data3 = b"compress test";
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(data3);
|
||||
let etag = hasher.finalize();
|
||||
let etag_hex = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
let reader3 = BufReader::new(Cursor::new(&data3[..]));
|
||||
let reader3 = Box::new(WarpReader::new(reader3));
|
||||
let etag_reader3 = EtagReader::new(reader3, Some(etag_hex.clone()));
|
||||
let etag_reader3 = EtagReader::new(reader3, Some("compress_wrapped_etag".to_string()));
|
||||
let mut compress_reader = CompressReader::new(etag_reader3, CompressionAlgorithm::Zstd);
|
||||
compress_reader.read_to_end(&mut Vec::new()).await.unwrap();
|
||||
assert_eq!(resolve_etag_generic(&mut compress_reader), Some(etag_hex.clone()));
|
||||
assert_eq!(resolve_etag_generic(&mut compress_reader), Some("compress_wrapped_etag".to_string()));
|
||||
|
||||
// Test 4: Double wrapper - CompressReader<EncryptReader<EtagReader>>
|
||||
let data4 = b"double wrap test";
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(data4);
|
||||
let etag = hasher.finalize();
|
||||
let etag_hex = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
let reader4 = BufReader::new(Cursor::new(&data4[..]));
|
||||
let reader4 = Box::new(WarpReader::new(reader4));
|
||||
let etag_reader4 = EtagReader::new(reader4, Some(etag_hex.clone()));
|
||||
let etag_reader4 = EtagReader::new(reader4, Some("double_wrapped_etag".to_string()));
|
||||
let key = [1u8; 32];
|
||||
let nonce = [1u8; 12];
|
||||
let encrypt_reader4 = EncryptReader::new(etag_reader4, key, nonce);
|
||||
let mut compress_reader4 = CompressReader::new(encrypt_reader4, CompressionAlgorithm::Gzip);
|
||||
compress_reader4.read_to_end(&mut Vec::new()).await.unwrap();
|
||||
assert_eq!(resolve_etag_generic(&mut compress_reader4), Some(etag_hex.clone()));
|
||||
assert_eq!(resolve_etag_generic(&mut compress_reader4), Some("double_wrapped_etag".to_string()));
|
||||
|
||||
println!("✅ All ETag extraction methods work correctly!");
|
||||
println!("✅ Trait-based approach handles recursive unwrapping!");
|
||||
@@ -235,7 +195,6 @@ mod tests {
|
||||
data.len() as i64,
|
||||
data.len() as i64,
|
||||
Some("real_world_etag".to_string()),
|
||||
None,
|
||||
false,
|
||||
)
|
||||
.unwrap();
|
||||
@@ -280,7 +239,7 @@ mod tests {
|
||||
let data = b"no etag test";
|
||||
let reader = BufReader::new(Cursor::new(&data[..]));
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
let mut hash_reader_no_etag = HashReader::new(reader, data.len() as i64, data.len() as i64, None, None, false).unwrap();
|
||||
let mut hash_reader_no_etag = HashReader::new(reader, data.len() as i64, data.len() as i64, None, false).unwrap();
|
||||
assert_eq!(resolve_etag_generic(&mut hash_reader_no_etag), None);
|
||||
|
||||
// Test with EtagReader that has None etag
|
||||
|
||||
@@ -19,7 +19,6 @@ use pin_project_lite::pin_project;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tracing::error;
|
||||
|
||||
pin_project! {
|
||||
pub struct EtagReader {
|
||||
@@ -44,8 +43,7 @@ impl EtagReader {
|
||||
/// Get the final md5 value (etag) as a hex string, only compute once.
|
||||
/// Can be called multiple times, always returns the same result after finished.
|
||||
pub fn get_etag(&mut self) -> String {
|
||||
let etag = self.md5.clone().finalize().to_vec();
|
||||
hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower)
|
||||
format!("{:x}", self.md5.clone().finalize())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -62,10 +60,8 @@ impl AsyncRead for EtagReader {
|
||||
// EOF
|
||||
*this.finished = true;
|
||||
if let Some(checksum) = this.checksum {
|
||||
let etag = this.md5.clone().finalize().to_vec();
|
||||
let etag_hex = hex_simd::encode_to_string(etag, hex_simd::AsciiCase::Lower);
|
||||
if *checksum != etag_hex {
|
||||
error!("Checksum mismatch, expected={:?}, actual={:?}", checksum, etag_hex);
|
||||
let etag = format!("{:x}", this.md5.clone().finalize());
|
||||
if *checksum != etag {
|
||||
return Poll::Ready(Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Checksum mismatch")));
|
||||
}
|
||||
}
|
||||
@@ -218,7 +214,7 @@ mod tests {
|
||||
let data = b"checksum test data";
|
||||
let mut hasher = Md5::new();
|
||||
hasher.update(data);
|
||||
let expected = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower);
|
||||
let expected = format!("{:x}", hasher.finalize());
|
||||
let reader = BufReader::new(&data[..]);
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
let mut etag_reader = EtagReader::new(reader, Some(expected.clone()));
|
||||
@@ -237,7 +233,7 @@ mod tests {
|
||||
let wrong_checksum = "deadbeefdeadbeefdeadbeefdeadbeef".to_string();
|
||||
let reader = BufReader::new(&data[..]);
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
let mut etag_reader = EtagReader::new(reader, Some(wrong_checksum.clone()));
|
||||
let mut etag_reader = EtagReader::new(reader, Some(wrong_checksum));
|
||||
|
||||
let mut buf = Vec::new();
|
||||
// Verification failed, should return InvalidData error
|
||||
|
||||
@@ -50,7 +50,7 @@
|
||||
//! let diskable_md5 = false;
|
||||
//!
|
||||
//! // Method 1: Simple creation (recommended for most cases)
|
||||
//! let hash_reader = HashReader::new(reader, size, actual_size, etag.clone(), None, diskable_md5).unwrap();
|
||||
//! let hash_reader = HashReader::new(reader, size, actual_size, etag.clone(), diskable_md5).unwrap();
|
||||
//!
|
||||
//! // Method 2: With manual wrapping to recreate original logic
|
||||
//! let reader2 = BufReader::new(Cursor::new(&data[..]));
|
||||
@@ -71,7 +71,7 @@
|
||||
//! // No wrapping needed
|
||||
//! reader2
|
||||
//! };
|
||||
//! let hash_reader2 = HashReader::new(wrapped_reader, size, actual_size, etag.clone(), None, diskable_md5).unwrap();
|
||||
//! let hash_reader2 = HashReader::new(wrapped_reader, size, actual_size, etag, diskable_md5).unwrap();
|
||||
//! # });
|
||||
//! ```
|
||||
//!
|
||||
@@ -88,43 +88,28 @@
|
||||
//! # tokio_test::block_on(async {
|
||||
//! let data = b"test";
|
||||
//! let reader = BufReader::new(Cursor::new(&data[..]));
|
||||
//! let hash_reader = HashReader::new(Box::new(WarpReader::new(reader)), 4, 4, None, None,false).unwrap();
|
||||
//! let hash_reader = HashReader::new(Box::new(WarpReader::new(reader)), 4, 4, None, false).unwrap();
|
||||
//!
|
||||
//! // Check if a type is a HashReader
|
||||
//! assert!(hash_reader.is_hash_reader());
|
||||
//!
|
||||
//! // Use new for compatibility (though it's simpler to use new() directly)
|
||||
//! let reader2 = BufReader::new(Cursor::new(&data[..]));
|
||||
//! let result = HashReader::new(Box::new(WarpReader::new(reader2)), 4, 4, None, None, false);
|
||||
//! let result = HashReader::new(Box::new(WarpReader::new(reader2)), 4, 4, None, false);
|
||||
//! assert!(result.is_ok());
|
||||
//! # });
|
||||
//! ```
|
||||
|
||||
use crate::Checksum;
|
||||
use crate::ChecksumHasher;
|
||||
use crate::ChecksumType;
|
||||
use crate::Sha256Hasher;
|
||||
use crate::compress_index::{Index, TryGetIndex};
|
||||
use crate::get_content_checksum;
|
||||
use crate::{EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, Reader, WarpReader};
|
||||
use base64::Engine;
|
||||
use base64::engine::general_purpose;
|
||||
use http::HeaderMap;
|
||||
use pin_project_lite::pin_project;
|
||||
use s3s::TrailingHeaders;
|
||||
use std::collections::HashMap;
|
||||
use std::io::Cursor;
|
||||
use std::io::Write;
|
||||
use std::mem;
|
||||
use std::pin::Pin;
|
||||
use std::task::{Context, Poll};
|
||||
use tokio::io::{AsyncRead, ReadBuf};
|
||||
use tracing::error;
|
||||
|
||||
use crate::compress_index::{Index, TryGetIndex};
|
||||
use crate::{EtagReader, EtagResolvable, HardLimitReader, HashReaderDetector, Reader};
|
||||
|
||||
/// Trait for mutable operations on HashReader
|
||||
pub trait HashReaderMut {
|
||||
fn into_inner(self) -> Box<dyn Reader>;
|
||||
fn take_inner(&mut self) -> Box<dyn Reader>;
|
||||
fn bytes_read(&self) -> u64;
|
||||
fn checksum(&self) -> &Option<String>;
|
||||
fn set_checksum(&mut self, checksum: Option<String>);
|
||||
@@ -132,10 +117,6 @@ pub trait HashReaderMut {
|
||||
fn set_size(&mut self, size: i64);
|
||||
fn actual_size(&self) -> i64;
|
||||
fn set_actual_size(&mut self, actual_size: i64);
|
||||
fn content_hash(&self) -> &Option<Checksum>;
|
||||
fn content_sha256(&self) -> &Option<String>;
|
||||
fn get_trailer(&self) -> Option<&TrailingHeaders>;
|
||||
fn set_trailer(&mut self, trailer: Option<TrailingHeaders>);
|
||||
}
|
||||
|
||||
pin_project! {
|
||||
@@ -148,14 +129,7 @@ pin_project! {
|
||||
pub actual_size: i64,
|
||||
pub diskable_md5: bool,
|
||||
bytes_read: u64,
|
||||
content_hash: Option<Checksum>,
|
||||
content_hasher: Option<Box<dyn ChecksumHasher>>,
|
||||
content_sha256: Option<String>,
|
||||
content_sha256_hasher: Option<Sha256Hasher>,
|
||||
checksum_on_finish: bool,
|
||||
|
||||
trailer_s3s: Option<TrailingHeaders>,
|
||||
|
||||
// TODO: content_hash
|
||||
}
|
||||
|
||||
}
|
||||
@@ -165,8 +139,7 @@ impl HashReader {
|
||||
mut inner: Box<dyn Reader>,
|
||||
size: i64,
|
||||
actual_size: i64,
|
||||
md5hex: Option<String>,
|
||||
sha256hex: Option<String>,
|
||||
md5: Option<String>,
|
||||
diskable_md5: bool,
|
||||
) -> std::io::Result<Self> {
|
||||
// Check if it's already a HashReader and update its parameters
|
||||
@@ -179,7 +152,7 @@ impl HashReader {
|
||||
}
|
||||
|
||||
if let Some(checksum) = existing_hash_reader.checksum() {
|
||||
if let Some(ref md5) = md5hex {
|
||||
if let Some(ref md5) = md5 {
|
||||
if checksum != md5 {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "HashReader checksum mismatch"));
|
||||
}
|
||||
@@ -193,7 +166,7 @@ impl HashReader {
|
||||
));
|
||||
}
|
||||
|
||||
existing_hash_reader.set_checksum(md5hex.clone());
|
||||
existing_hash_reader.set_checksum(md5.clone());
|
||||
|
||||
if existing_hash_reader.size() < 0 && size >= 0 {
|
||||
existing_hash_reader.set_size(size);
|
||||
@@ -203,29 +176,13 @@ impl HashReader {
|
||||
existing_hash_reader.set_actual_size(actual_size);
|
||||
}
|
||||
|
||||
let size = existing_hash_reader.size();
|
||||
let actual_size = existing_hash_reader.actual_size();
|
||||
let content_hash = existing_hash_reader.content_hash().clone();
|
||||
let content_hasher = existing_hash_reader
|
||||
.content_hash()
|
||||
.clone()
|
||||
.map(|hash| hash.checksum_type.hasher().unwrap());
|
||||
let content_sha256 = existing_hash_reader.content_sha256().clone();
|
||||
let content_sha256_hasher = existing_hash_reader.content_sha256().clone().map(|_| Sha256Hasher::new());
|
||||
let inner = existing_hash_reader.take_inner();
|
||||
return Ok(Self {
|
||||
inner,
|
||||
size,
|
||||
checksum: md5hex.clone(),
|
||||
checksum: md5,
|
||||
actual_size,
|
||||
diskable_md5,
|
||||
bytes_read: 0,
|
||||
content_sha256,
|
||||
content_sha256_hasher,
|
||||
content_hash,
|
||||
content_hasher,
|
||||
checksum_on_finish: false,
|
||||
trailer_s3s: existing_hash_reader.get_trailer().cloned(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -233,33 +190,23 @@ impl HashReader {
|
||||
let hr = HardLimitReader::new(inner, size);
|
||||
inner = Box::new(hr);
|
||||
if !diskable_md5 && !inner.is_hash_reader() {
|
||||
let er = EtagReader::new(inner, md5hex.clone());
|
||||
let er = EtagReader::new(inner, md5.clone());
|
||||
inner = Box::new(er);
|
||||
}
|
||||
} else if !diskable_md5 {
|
||||
let er = EtagReader::new(inner, md5hex.clone());
|
||||
let er = EtagReader::new(inner, md5.clone());
|
||||
inner = Box::new(er);
|
||||
}
|
||||
Ok(Self {
|
||||
inner,
|
||||
size,
|
||||
checksum: md5hex,
|
||||
checksum: md5,
|
||||
actual_size,
|
||||
diskable_md5,
|
||||
bytes_read: 0,
|
||||
content_hash: None,
|
||||
content_hasher: None,
|
||||
content_sha256: sha256hex.clone(),
|
||||
content_sha256_hasher: sha256hex.clone().map(|_| Sha256Hasher::new()),
|
||||
checksum_on_finish: false,
|
||||
trailer_s3s: None,
|
||||
})
|
||||
}
|
||||
|
||||
pub fn into_inner(self) -> Box<dyn Reader> {
|
||||
self.inner
|
||||
}
|
||||
|
||||
/// Update HashReader parameters
|
||||
pub fn update_params(&mut self, size: i64, actual_size: i64, etag: Option<String>) {
|
||||
if self.size < 0 && size >= 0 {
|
||||
@@ -281,112 +228,9 @@ impl HashReader {
|
||||
pub fn actual_size(&self) -> i64 {
|
||||
self.actual_size
|
||||
}
|
||||
|
||||
pub fn add_checksum_from_s3s(
|
||||
&mut self,
|
||||
headers: &HeaderMap,
|
||||
trailing_headers: Option<TrailingHeaders>,
|
||||
ignore_value: bool,
|
||||
) -> Result<(), std::io::Error> {
|
||||
let cs = get_content_checksum(headers)?;
|
||||
|
||||
if ignore_value {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(checksum) = cs {
|
||||
if checksum.checksum_type.trailing() {
|
||||
self.trailer_s3s = trailing_headers.clone();
|
||||
}
|
||||
|
||||
self.content_hash = Some(checksum.clone());
|
||||
|
||||
return self.add_non_trailing_checksum(Some(checksum), ignore_value);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add_checksum_no_trailer(&mut self, header: &HeaderMap, ignore_value: bool) -> Result<(), std::io::Error> {
|
||||
let cs = get_content_checksum(header)?;
|
||||
|
||||
if let Some(checksum) = cs {
|
||||
self.content_hash = Some(checksum.clone());
|
||||
|
||||
return self.add_non_trailing_checksum(Some(checksum), ignore_value);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn add_non_trailing_checksum(&mut self, checksum: Option<Checksum>, ignore_value: bool) -> Result<(), std::io::Error> {
|
||||
if let Some(checksum) = checksum {
|
||||
self.content_hash = Some(checksum.clone());
|
||||
|
||||
if ignore_value {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
if let Some(hasher) = checksum.checksum_type.hasher() {
|
||||
self.content_hasher = Some(hasher);
|
||||
} else {
|
||||
return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid checksum type"));
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
pub fn checksum(&self) -> Option<Checksum> {
|
||||
if self
|
||||
.content_hash
|
||||
.as_ref()
|
||||
.is_none_or(|v| !v.checksum_type.is_set() || !v.valid())
|
||||
{
|
||||
return None;
|
||||
}
|
||||
self.content_hash.clone()
|
||||
}
|
||||
pub fn content_crc_type(&self) -> Option<ChecksumType> {
|
||||
self.content_hash.as_ref().map(|v| v.checksum_type)
|
||||
}
|
||||
|
||||
pub fn content_crc(&self) -> HashMap<String, String> {
|
||||
let mut map = HashMap::new();
|
||||
if let Some(checksum) = self.content_hash.as_ref() {
|
||||
if !checksum.valid() || checksum.checksum_type.is(ChecksumType::NONE) {
|
||||
return map;
|
||||
}
|
||||
|
||||
if checksum.checksum_type.trailing() {
|
||||
if let Some(trailer) = self.trailer_s3s.as_ref() {
|
||||
if let Some(Some(checksum_str)) = trailer.read(|headers| {
|
||||
headers
|
||||
.get(checksum.checksum_type.to_string())
|
||||
.and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
||||
}) {
|
||||
map.insert(checksum.checksum_type.to_string(), checksum_str);
|
||||
}
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
map.insert(checksum.checksum_type.to_string(), checksum.encoded.clone());
|
||||
|
||||
return map;
|
||||
}
|
||||
map
|
||||
}
|
||||
}
|
||||
|
||||
impl HashReaderMut for HashReader {
|
||||
fn into_inner(self) -> Box<dyn Reader> {
|
||||
self.inner
|
||||
}
|
||||
|
||||
fn take_inner(&mut self) -> Box<dyn Reader> {
|
||||
// Replace inner with an empty reader to move it out safely while keeping self valid
|
||||
mem::replace(&mut self.inner, Box::new(WarpReader::new(Cursor::new(Vec::new()))))
|
||||
}
|
||||
|
||||
fn bytes_read(&self) -> u64 {
|
||||
self.bytes_read
|
||||
}
|
||||
@@ -414,108 +258,22 @@ impl HashReaderMut for HashReader {
|
||||
fn set_actual_size(&mut self, actual_size: i64) {
|
||||
self.actual_size = actual_size;
|
||||
}
|
||||
|
||||
fn content_hash(&self) -> &Option<Checksum> {
|
||||
&self.content_hash
|
||||
}
|
||||
|
||||
fn content_sha256(&self) -> &Option<String> {
|
||||
&self.content_sha256
|
||||
}
|
||||
|
||||
fn get_trailer(&self) -> Option<&TrailingHeaders> {
|
||||
self.trailer_s3s.as_ref()
|
||||
}
|
||||
|
||||
fn set_trailer(&mut self, trailer: Option<TrailingHeaders>) {
|
||||
self.trailer_s3s = trailer;
|
||||
}
|
||||
}
|
||||
|
||||
impl AsyncRead for HashReader {
|
||||
fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll<std::io::Result<()>> {
|
||||
let this = self.project();
|
||||
let poll = this.inner.poll_read(cx, buf);
|
||||
if let Poll::Ready(Ok(())) = &poll {
|
||||
let filled = buf.filled().len();
|
||||
*this.bytes_read += filled as u64;
|
||||
|
||||
let before = buf.filled().len();
|
||||
match this.inner.poll_read(cx, buf) {
|
||||
Poll::Pending => Poll::Pending,
|
||||
Poll::Ready(Ok(())) => {
|
||||
let data = &buf.filled()[before..];
|
||||
let filled = data.len();
|
||||
|
||||
*this.bytes_read += filled as u64;
|
||||
|
||||
if filled > 0 {
|
||||
// Update SHA256 hasher
|
||||
if let Some(hasher) = this.content_sha256_hasher {
|
||||
if let Err(e) = hasher.write_all(data) {
|
||||
error!("SHA256 hasher write error, error={:?}", e);
|
||||
return Poll::Ready(Err(std::io::Error::other(e)));
|
||||
}
|
||||
}
|
||||
|
||||
// Update content hasher
|
||||
if let Some(hasher) = this.content_hasher {
|
||||
if let Err(e) = hasher.write_all(data) {
|
||||
return Poll::Ready(Err(std::io::Error::other(e)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if filled == 0 && !*this.checksum_on_finish {
|
||||
// check SHA256
|
||||
if let (Some(hasher), Some(expected_sha256)) = (this.content_sha256_hasher, this.content_sha256) {
|
||||
let sha256 = hex_simd::encode_to_string(hasher.finalize(), hex_simd::AsciiCase::Lower);
|
||||
if sha256 != *expected_sha256 {
|
||||
error!("SHA256 mismatch, expected={:?}, actual={:?}", expected_sha256, sha256);
|
||||
return Poll::Ready(Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "SHA256 mismatch")));
|
||||
}
|
||||
}
|
||||
|
||||
// check content hasher
|
||||
if let (Some(hasher), Some(expected_content_hash)) = (this.content_hasher, this.content_hash) {
|
||||
if expected_content_hash.checksum_type.trailing() {
|
||||
if let Some(trailer) = this.trailer_s3s.as_ref() {
|
||||
if let Some(Some(checksum_str)) = trailer.read(|headers| {
|
||||
expected_content_hash.checksum_type.key().and_then(|key| {
|
||||
headers.get(key).and_then(|value| value.to_str().ok().map(|s| s.to_string()))
|
||||
})
|
||||
}) {
|
||||
expected_content_hash.encoded = checksum_str;
|
||||
expected_content_hash.raw = general_purpose::STANDARD
|
||||
.decode(&expected_content_hash.encoded)
|
||||
.map_err(|_| std::io::Error::other("Invalid base64 checksum"))?;
|
||||
|
||||
if expected_content_hash.raw.is_empty() {
|
||||
return Poll::Ready(Err(std::io::Error::other("Content hash mismatch")));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let content_hash = hasher.finalize();
|
||||
|
||||
if content_hash != expected_content_hash.raw {
|
||||
error!(
|
||||
"Content hash mismatch, type={:?}, encoded={:?}, expected={:?}, actual={:?}",
|
||||
expected_content_hash.checksum_type,
|
||||
expected_content_hash.encoded,
|
||||
hex_simd::encode_to_string(&expected_content_hash.raw, hex_simd::AsciiCase::Lower),
|
||||
hex_simd::encode_to_string(content_hash, hex_simd::AsciiCase::Lower)
|
||||
);
|
||||
return Poll::Ready(Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
"Content hash mismatch",
|
||||
)));
|
||||
}
|
||||
}
|
||||
|
||||
*this.checksum_on_finish = true;
|
||||
}
|
||||
Poll::Ready(Ok(()))
|
||||
if filled == 0 {
|
||||
// EOF
|
||||
// TODO: check content_hash
|
||||
}
|
||||
Poll::Ready(Err(e)) => Poll::Ready(Err(e)),
|
||||
}
|
||||
poll
|
||||
}
|
||||
}
|
||||
|
||||
@@ -565,7 +323,7 @@ mod tests {
|
||||
// Test 1: Simple creation
|
||||
let reader1 = BufReader::new(Cursor::new(&data[..]));
|
||||
let reader1 = Box::new(WarpReader::new(reader1));
|
||||
let hash_reader1 = HashReader::new(reader1, size, actual_size, etag.clone(), None, false).unwrap();
|
||||
let hash_reader1 = HashReader::new(reader1, size, actual_size, etag.clone(), false).unwrap();
|
||||
assert_eq!(hash_reader1.size(), size);
|
||||
assert_eq!(hash_reader1.actual_size(), actual_size);
|
||||
|
||||
@@ -574,7 +332,7 @@ mod tests {
|
||||
let reader2 = Box::new(WarpReader::new(reader2));
|
||||
let hard_limit = HardLimitReader::new(reader2, size);
|
||||
let hard_limit = Box::new(hard_limit);
|
||||
let hash_reader2 = HashReader::new(hard_limit, size, actual_size, etag.clone(), None, false).unwrap();
|
||||
let hash_reader2 = HashReader::new(hard_limit, size, actual_size, etag.clone(), false).unwrap();
|
||||
assert_eq!(hash_reader2.size(), size);
|
||||
assert_eq!(hash_reader2.actual_size(), actual_size);
|
||||
|
||||
@@ -583,7 +341,7 @@ mod tests {
|
||||
let reader3 = Box::new(WarpReader::new(reader3));
|
||||
let etag_reader = EtagReader::new(reader3, etag.clone());
|
||||
let etag_reader = Box::new(etag_reader);
|
||||
let hash_reader3 = HashReader::new(etag_reader, size, actual_size, etag.clone(), None, false).unwrap();
|
||||
let hash_reader3 = HashReader::new(etag_reader, size, actual_size, etag.clone(), false).unwrap();
|
||||
assert_eq!(hash_reader3.size(), size);
|
||||
assert_eq!(hash_reader3.actual_size(), actual_size);
|
||||
}
|
||||
@@ -593,7 +351,7 @@ mod tests {
|
||||
let data = b"hello hashreader";
|
||||
let reader = BufReader::new(Cursor::new(&data[..]));
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
let mut hash_reader = HashReader::new(reader, data.len() as i64, data.len() as i64, None, None, false).unwrap();
|
||||
let mut hash_reader = HashReader::new(reader, data.len() as i64, data.len() as i64, None, false).unwrap();
|
||||
let mut buf = Vec::new();
|
||||
let _ = hash_reader.read_to_end(&mut buf).await.unwrap();
|
||||
// Since we removed EtagReader integration, etag might be None
|
||||
@@ -607,7 +365,7 @@ mod tests {
|
||||
let data = b"no etag";
|
||||
let reader = BufReader::new(Cursor::new(&data[..]));
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
let mut hash_reader = HashReader::new(reader, data.len() as i64, data.len() as i64, None, None, true).unwrap();
|
||||
let mut hash_reader = HashReader::new(reader, data.len() as i64, data.len() as i64, None, true).unwrap();
|
||||
let mut buf = Vec::new();
|
||||
let _ = hash_reader.read_to_end(&mut buf).await.unwrap();
|
||||
// Etag should be None when diskable_md5 is true
|
||||
@@ -623,17 +381,10 @@ mod tests {
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
// Create a HashReader first
|
||||
let hash_reader =
|
||||
HashReader::new(reader, data.len() as i64, data.len() as i64, Some("test_etag".to_string()), None, false).unwrap();
|
||||
HashReader::new(reader, data.len() as i64, data.len() as i64, Some("test_etag".to_string()), false).unwrap();
|
||||
let hash_reader = Box::new(WarpReader::new(hash_reader));
|
||||
// Now try to create another HashReader from the existing one using new
|
||||
let result = HashReader::new(
|
||||
hash_reader,
|
||||
data.len() as i64,
|
||||
data.len() as i64,
|
||||
Some("test_etag".to_string()),
|
||||
None,
|
||||
false,
|
||||
);
|
||||
let result = HashReader::new(hash_reader, data.len() as i64, data.len() as i64, Some("test_etag".to_string()), false);
|
||||
|
||||
assert!(result.is_ok());
|
||||
let final_reader = result.unwrap();
|
||||
@@ -671,7 +422,7 @@ mod tests {
|
||||
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
// Create HashReader
|
||||
let mut hr = HashReader::new(reader, size, actual_size, Some(expected.clone()), None, false).unwrap();
|
||||
let mut hr = HashReader::new(reader, size, actual_size, Some(expected.clone()), false).unwrap();
|
||||
|
||||
// If compression is enabled, compress data first
|
||||
let compressed_data = if is_compress {
|
||||
@@ -767,7 +518,7 @@ mod tests {
|
||||
|
||||
let reader = BufReader::new(Cursor::new(data.clone()));
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
let hash_reader = HashReader::new(reader, data.len() as i64, data.len() as i64, None, None, false).unwrap();
|
||||
let hash_reader = HashReader::new(reader, data.len() as i64, data.len() as i64, None, false).unwrap();
|
||||
|
||||
// Test compression
|
||||
let compress_reader = CompressReader::new(hash_reader, CompressionAlgorithm::Gzip);
|
||||
@@ -813,7 +564,7 @@ mod tests {
|
||||
|
||||
let reader = BufReader::new(Cursor::new(data.clone()));
|
||||
let reader = Box::new(WarpReader::new(reader));
|
||||
let hash_reader = HashReader::new(reader, data.len() as i64, data.len() as i64, None, None, false).unwrap();
|
||||
let hash_reader = HashReader::new(reader, data.len() as i64, data.len() as i64, None, false).unwrap();
|
||||
|
||||
// Compress
|
||||
let compress_reader = CompressReader::new(hash_reader, algorithm);
|
||||
|
||||
@@ -34,11 +34,6 @@ pub use hardlimit_reader::HardLimitReader;
|
||||
|
||||
mod hash_reader;
|
||||
pub use hash_reader::*;
|
||||
mod checksum;
|
||||
pub use checksum::*;
|
||||
|
||||
mod errors;
|
||||
pub use errors::*;
|
||||
|
||||
pub mod reader;
|
||||
pub use reader::WarpReader;
|
||||
|
||||
@@ -34,7 +34,6 @@ hyper.workspace = true
|
||||
serde_urlencoded.workspace = true
|
||||
rustfs-utils = { workspace = true, features = ["full"] }
|
||||
s3s.workspace = true
|
||||
base64-simd.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
|
||||
|
||||
@@ -20,7 +20,7 @@ use std::fmt::Write;
|
||||
use time::{OffsetDateTime, format_description};
|
||||
|
||||
use super::utils::get_host_addr;
|
||||
use rustfs_utils::crypto::{hex, hmac_sha1};
|
||||
use rustfs_utils::crypto::{base64_encode, hex, hmac_sha1};
|
||||
use s3s::Body;
|
||||
|
||||
const _SIGN_V4_ALGORITHM: &str = "AWS4-HMAC-SHA256";
|
||||
@@ -111,11 +111,7 @@ pub fn sign_v2(
|
||||
}
|
||||
|
||||
let auth_header = format!("{SIGN_V2_ALGORITHM} {access_key_id}:");
|
||||
let auth_header = format!(
|
||||
"{}{}",
|
||||
auth_header,
|
||||
base64_simd::URL_SAFE_NO_PAD.encode_to_string(hmac_sha1(secret_access_key, string_to_sign))
|
||||
);
|
||||
let auth_header = format!("{}{}", auth_header, base64_encode(&hmac_sha1(secret_access_key, string_to_sign)));
|
||||
|
||||
headers.insert("Authorization", auth_header.parse().unwrap());
|
||||
|
||||
|
||||
@@ -37,7 +37,6 @@ hex-simd = { workspace = true, optional = true }
|
||||
highway = { workspace = true, optional = true }
|
||||
hickory-resolver = { workspace = true, optional = true }
|
||||
hmac = { workspace = true, optional = true }
|
||||
http = { workspace = true, optional = true }
|
||||
hyper = { workspace = true, optional = true }
|
||||
libc = { workspace = true, optional = true }
|
||||
local-ip-address = { workspace = true, optional = true }
|
||||
@@ -94,5 +93,5 @@ hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:s
|
||||
os = ["dep:nix", "dep:tempfile", "winapi"] # operating system utilities
|
||||
integration = [] # integration test features
|
||||
sys = ["dep:sysinfo"] # system information features
|
||||
http = ["dep:convert_case", "dep:http"]
|
||||
http = ["dep:convert_case"]
|
||||
full = ["ip", "tls", "net", "io", "hash", "os", "integration", "path", "crypto", "string", "compress", "sys", "notify", "http"] # all features
|
||||
|
||||
@@ -102,7 +102,7 @@ pub fn load_all_certs_from_directory(
|
||||
let path = entry.path();
|
||||
|
||||
if path.is_dir() {
|
||||
let domain_name: &str = path
|
||||
let domain_name = path
|
||||
.file_name()
|
||||
.and_then(|name| name.to_str())
|
||||
.ok_or_else(|| certs_error(format!("invalid domain name directory:{path:?}")))?;
|
||||
|
||||
@@ -17,11 +17,11 @@ use std::mem::MaybeUninit;
|
||||
use hex_simd::{AsOut, AsciiCase};
|
||||
use hyper::body::Bytes;
|
||||
|
||||
pub fn base64_encode_url_safe_no_pad(input: &[u8]) -> String {
|
||||
pub fn base64_encode(input: &[u8]) -> String {
|
||||
base64_simd::URL_SAFE_NO_PAD.encode_to_string(input)
|
||||
}
|
||||
|
||||
pub fn base64_decode_url_safe_no_pad(input: &[u8]) -> Result<Vec<u8>, base64_simd::Error> {
|
||||
pub fn base64_decode(input: &[u8]) -> Result<Vec<u8>, base64_simd::Error> {
|
||||
base64_simd::URL_SAFE_NO_PAD.decode_to_vec(input)
|
||||
}
|
||||
|
||||
@@ -89,11 +89,11 @@ pub fn hex_sha256_chunk<R>(chunk: &[Bytes], f: impl FnOnce(&str) -> R) -> R {
|
||||
fn test_base64_encoding_decoding() {
|
||||
let original_uuid_timestamp = "c0194290-d911-45cb-8e12-79ec563f46a8x1735460504394878000";
|
||||
|
||||
let encoded_string = base64_encode_url_safe_no_pad(original_uuid_timestamp.as_bytes());
|
||||
let encoded_string = base64_encode(original_uuid_timestamp.as_bytes());
|
||||
|
||||
println!("Encoded: {}", &encoded_string);
|
||||
|
||||
let decoded_bytes = base64_decode_url_safe_no_pad(encoded_string.clone().as_bytes()).unwrap();
|
||||
let decoded_bytes = base64_decode(encoded_string.clone().as_bytes()).unwrap();
|
||||
let decoded_string = String::from_utf8(decoded_bytes).unwrap();
|
||||
|
||||
assert_eq!(decoded_string, original_uuid_timestamp)
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user