mirror of
https://github.com/rustfs/rustfs.git
synced 2026-01-16 17:20:33 +00:00
refactor: Restructure project layout and clean up dependencies (#30)
This commit introduces a significant reorganization of the project structure to improve maintainability and clarity. Key changes include: - Adjusted the directory layout for a more logical module organization. - Removed unused crate dependencies, reducing the overall project size and potentially speeding up build times. - Updated import paths and configuration files to reflect the structural changes.
This commit is contained in:
houseme
656
Cargo.lock
generated
656
Cargo.lock
generated
@@ -173,43 +173,6 @@ version = "1.0.98"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487"
|
||||
|
||||
[[package]]
|
||||
name = "api"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"common",
|
||||
"datafusion",
|
||||
"ecstore",
|
||||
"futures",
|
||||
"futures-core",
|
||||
"http 1.3.1",
|
||||
"object_store",
|
||||
"pin-project-lite",
|
||||
"s3s",
|
||||
"snafu",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"transform-stream",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "appauth"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"common",
|
||||
"hex-simd",
|
||||
"rand 0.9.1",
|
||||
"rsa",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arbitrary"
|
||||
version = "1.4.1"
|
||||
@@ -653,20 +616,6 @@ dependencies = [
|
||||
"syn 2.0.101",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "async_zip"
|
||||
version = "0.0.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "00b9f7252833d5ed4b00aa9604b563529dd5e11de9c23615de2dcdf91eb87b52"
|
||||
dependencies = [
|
||||
"crc32fast",
|
||||
"futures-lite",
|
||||
"pin-project",
|
||||
"thiserror 1.0.69",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "atk"
|
||||
version = "0.18.2"
|
||||
@@ -1142,17 +1091,6 @@ dependencies = [
|
||||
"tower-service",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backon"
|
||||
version = "1.5.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "302eaff5357a264a2c42f127ecb8bac761cf99749fc3dc95677e2743991f99e7"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"gloo-timers",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "backtrace"
|
||||
version = "0.3.75"
|
||||
@@ -1838,18 +1776,6 @@ dependencies = [
|
||||
"unicode-width",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "common"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"lazy_static",
|
||||
"scopeguard",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"tracing-error",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "concurrent-queue"
|
||||
version = "2.5.0"
|
||||
@@ -2208,24 +2134,6 @@ version = "0.2.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929"
|
||||
|
||||
[[package]]
|
||||
name = "crypto"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
"cfg-if",
|
||||
"chacha20poly1305",
|
||||
"jsonwebtoken",
|
||||
"pbkdf2",
|
||||
"rand 0.9.1",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"test-case",
|
||||
"thiserror 2.0.12",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.4.9"
|
||||
@@ -3566,21 +3474,18 @@ name = "e2e_test"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"common",
|
||||
"ecstore",
|
||||
"flatbuffers 25.2.10",
|
||||
"futures",
|
||||
"lazy_static",
|
||||
"lock",
|
||||
"madmin",
|
||||
"protos",
|
||||
"rmp-serde",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-lock",
|
||||
"rustfs-madmin",
|
||||
"rustfs-protos",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"tower",
|
||||
"url",
|
||||
]
|
||||
|
||||
@@ -3596,81 +3501,6 @@ dependencies = [
|
||||
"signature 1.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ecstore"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"async-channel",
|
||||
"async-trait",
|
||||
"aws-sdk-s3",
|
||||
"base64 0.22.1",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"bytesize",
|
||||
"chrono",
|
||||
"common",
|
||||
"criterion",
|
||||
"enumset",
|
||||
"flatbuffers 25.2.10",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"glob",
|
||||
"hex-simd",
|
||||
"hmac 0.12.1",
|
||||
"http 1.3.1",
|
||||
"hyper 1.6.0",
|
||||
"hyper-rustls 0.27.7",
|
||||
"hyper-util",
|
||||
"lazy_static",
|
||||
"lock",
|
||||
"madmin",
|
||||
"md-5",
|
||||
"nix 0.30.1",
|
||||
"num_cpus",
|
||||
"once_cell",
|
||||
"path-absolutize",
|
||||
"path-clean",
|
||||
"pin-project-lite",
|
||||
"policy",
|
||||
"protos",
|
||||
"rand 0.9.1",
|
||||
"reed-solomon-simd",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rmp",
|
||||
"rmp-serde",
|
||||
"rustfs-config",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-rio",
|
||||
"rustfs-rsc",
|
||||
"rustfs-signer",
|
||||
"rustfs-utils",
|
||||
"rustls 0.23.28",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde-xml-rs 0.8.1",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"shadow-rs",
|
||||
"smallvec",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.12",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tonic",
|
||||
"tower",
|
||||
"tracing",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
"winapi",
|
||||
"workers",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "either"
|
||||
version = "1.15.0"
|
||||
@@ -4446,18 +4276,6 @@ dependencies = [
|
||||
"web-sys",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gloo-timers"
|
||||
version = "0.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994"
|
||||
dependencies = [
|
||||
"futures-channel",
|
||||
"futures-core",
|
||||
"js-sys",
|
||||
"wasm-bindgen",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "gloo-utils"
|
||||
version = "0.2.0"
|
||||
@@ -4904,36 +4722,6 @@ dependencies = [
|
||||
"windows-registry",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iam"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
"common",
|
||||
"crypto",
|
||||
"ecstore",
|
||||
"futures",
|
||||
"ipnetwork",
|
||||
"itertools 0.14.0",
|
||||
"jsonwebtoken",
|
||||
"lazy_static",
|
||||
"madmin",
|
||||
"policy",
|
||||
"rand 0.9.1",
|
||||
"regex",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"test-case",
|
||||
"thiserror 2.0.12",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "iana-time-zone"
|
||||
version = "0.1.63"
|
||||
@@ -5087,25 +4875,6 @@ dependencies = [
|
||||
"winapi-util",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "include_dir"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd"
|
||||
dependencies = [
|
||||
"include_dir_macros",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "include_dir_macros"
|
||||
version = "0.7.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "indexmap"
|
||||
version = "1.9.3"
|
||||
@@ -5662,26 +5431,6 @@ dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"backon",
|
||||
"common",
|
||||
"lazy_static",
|
||||
"protos",
|
||||
"rand 0.9.1",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"tracing",
|
||||
"tracing-error",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "lock_api"
|
||||
version = "0.4.13"
|
||||
@@ -5778,21 +5527,6 @@ version = "0.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4"
|
||||
|
||||
[[package]]
|
||||
name = "madmin"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"common",
|
||||
"humantime",
|
||||
"hyper 1.6.0",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "malloc_buf"
|
||||
version = "0.0.6"
|
||||
@@ -7202,33 +6936,6 @@ dependencies = [
|
||||
"miniz_oxide",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "policy"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
"common",
|
||||
"crypto",
|
||||
"futures",
|
||||
"ipnetwork",
|
||||
"itertools 0.14.0",
|
||||
"jsonwebtoken",
|
||||
"lazy_static",
|
||||
"madmin",
|
||||
"rand 0.9.1",
|
||||
"regex",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"test-case",
|
||||
"thiserror 2.0.12",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "polling"
|
||||
version = "3.8.0"
|
||||
@@ -7450,41 +7157,6 @@ dependencies = [
|
||||
"prost",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf"
|
||||
version = "3.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"protobuf-support",
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protobuf-support"
|
||||
version = "3.7.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6"
|
||||
dependencies = [
|
||||
"thiserror 1.0.69",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "protos"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"common",
|
||||
"flatbuffers 25.2.10",
|
||||
"prost",
|
||||
"prost-build",
|
||||
"protobuf",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"tonic-build",
|
||||
"tower",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "psm"
|
||||
version = "0.1.26"
|
||||
@@ -7494,24 +7166,6 @@ dependencies = [
|
||||
"cc",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "query"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"api",
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
"datafusion",
|
||||
"derive_builder",
|
||||
"futures",
|
||||
"lazy_static",
|
||||
"parking_lot",
|
||||
"s3s",
|
||||
"snafu",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "quick-xml"
|
||||
version = "0.37.5"
|
||||
@@ -8178,59 +7832,45 @@ dependencies = [
|
||||
name = "rustfs"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"api",
|
||||
"appauth",
|
||||
"async-trait",
|
||||
"atoi",
|
||||
"atomic_enum",
|
||||
"aws-sdk-s3",
|
||||
"axum",
|
||||
"axum-extra",
|
||||
"axum-server",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"clap",
|
||||
"common",
|
||||
"const-str",
|
||||
"crypto",
|
||||
"datafusion",
|
||||
"ecstore",
|
||||
"flatbuffers 25.2.10",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"hmac 0.12.1",
|
||||
"http 1.3.1",
|
||||
"http-body 1.0.1",
|
||||
"hyper 1.6.0",
|
||||
"hyper-util",
|
||||
"iam",
|
||||
"include_dir",
|
||||
"jsonwebtoken",
|
||||
"lazy_static",
|
||||
"libsystemd",
|
||||
"lock",
|
||||
"madmin",
|
||||
"matchit",
|
||||
"mime",
|
||||
"mime_guess",
|
||||
"netif",
|
||||
"opentelemetry",
|
||||
"percent-encoding",
|
||||
"pin-project-lite",
|
||||
"policy",
|
||||
"prost-build",
|
||||
"protos",
|
||||
"query",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rmp-serde",
|
||||
"rust-embed",
|
||||
"rustfs-appauth",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-iam",
|
||||
"rustfs-madmin",
|
||||
"rustfs-notify",
|
||||
"rustfs-obs",
|
||||
"rustfs-policy",
|
||||
"rustfs-protos",
|
||||
"rustfs-rio",
|
||||
"rustfs-s3select-api",
|
||||
"rustfs-s3select-query",
|
||||
"rustfs-utils",
|
||||
"rustfs-zip",
|
||||
"rustls 0.23.28",
|
||||
@@ -8238,7 +7878,6 @@ dependencies = [
|
||||
"serde",
|
||||
"serde_json",
|
||||
"serde_urlencoded",
|
||||
"sha2 0.10.9",
|
||||
"shadow-rs",
|
||||
"socket2",
|
||||
"thiserror 2.0.12",
|
||||
@@ -8250,16 +7889,33 @@ dependencies = [
|
||||
"tokio-tar",
|
||||
"tokio-util",
|
||||
"tonic",
|
||||
"tonic-build",
|
||||
"tower",
|
||||
"tower-http",
|
||||
"tracing",
|
||||
"transform-stream",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-appauth"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"rsa",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-common"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"tokio",
|
||||
"tonic",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-config"
|
||||
version = "0.0.1"
|
||||
@@ -8269,6 +7925,99 @@ dependencies = [
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-crypto"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"argon2",
|
||||
"cfg-if",
|
||||
"chacha20poly1305",
|
||||
"jsonwebtoken",
|
||||
"pbkdf2",
|
||||
"rand 0.9.1",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"test-case",
|
||||
"thiserror 2.0.12",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-ecstore"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"async-channel",
|
||||
"async-trait",
|
||||
"aws-sdk-s3",
|
||||
"base64 0.22.1",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"bytesize",
|
||||
"chrono",
|
||||
"criterion",
|
||||
"enumset",
|
||||
"flatbuffers 25.2.10",
|
||||
"futures",
|
||||
"futures-util",
|
||||
"glob",
|
||||
"hex-simd",
|
||||
"hmac 0.12.1",
|
||||
"http 1.3.1",
|
||||
"hyper 1.6.0",
|
||||
"hyper-rustls 0.27.7",
|
||||
"hyper-util",
|
||||
"lazy_static",
|
||||
"md-5",
|
||||
"nix 0.30.1",
|
||||
"num_cpus",
|
||||
"once_cell",
|
||||
"path-absolutize",
|
||||
"path-clean",
|
||||
"pin-project-lite",
|
||||
"rand 0.9.1",
|
||||
"reed-solomon-simd",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rmp",
|
||||
"rmp-serde",
|
||||
"rustfs-common",
|
||||
"rustfs-config",
|
||||
"rustfs-filemeta",
|
||||
"rustfs-lock",
|
||||
"rustfs-madmin",
|
||||
"rustfs-policy",
|
||||
"rustfs-protos",
|
||||
"rustfs-rio",
|
||||
"rustfs-rsc",
|
||||
"rustfs-signer",
|
||||
"rustfs-utils",
|
||||
"rustfs-workers",
|
||||
"rustls 0.23.28",
|
||||
"s3s",
|
||||
"serde",
|
||||
"serde-xml-rs 0.8.1",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"shadow-rs",
|
||||
"smallvec",
|
||||
"temp-env",
|
||||
"tempfile",
|
||||
"thiserror 2.0.12",
|
||||
"time",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-util",
|
||||
"tonic",
|
||||
"tower",
|
||||
"tracing",
|
||||
"url",
|
||||
"urlencoding",
|
||||
"uuid",
|
||||
"winapi",
|
||||
"xxhash-rust",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-filemeta"
|
||||
version = "0.0.1"
|
||||
@@ -8311,6 +8060,59 @@ dependencies = [
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-iam"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
"futures",
|
||||
"jsonwebtoken",
|
||||
"lazy_static",
|
||||
"rand 0.9.1",
|
||||
"rustfs-crypto",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-madmin",
|
||||
"rustfs-policy",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"thiserror 2.0.12",
|
||||
"time",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-lock"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"lazy_static",
|
||||
"rand 0.9.1",
|
||||
"rustfs-protos",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"tokio",
|
||||
"tonic",
|
||||
"tracing",
|
||||
"url",
|
||||
"uuid",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-madmin"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"chrono",
|
||||
"humantime",
|
||||
"hyper 1.6.0",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"time",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-notify"
|
||||
version = "0.0.1"
|
||||
@@ -8319,13 +8121,13 @@ dependencies = [
|
||||
"axum",
|
||||
"chrono",
|
||||
"dashmap 6.1.0",
|
||||
"ecstore",
|
||||
"form_urlencoded",
|
||||
"once_cell",
|
||||
"quick-xml",
|
||||
"reqwest",
|
||||
"rumqttc",
|
||||
"rustfs-config",
|
||||
"rustfs-ecstore",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
"serde_json",
|
||||
@@ -8373,19 +8175,45 @@ dependencies = [
|
||||
"tracing-subscriber",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-policy"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"ipnetwork",
|
||||
"jsonwebtoken",
|
||||
"rand 0.9.1",
|
||||
"regex",
|
||||
"rustfs-crypto",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"strum",
|
||||
"test-case",
|
||||
"thiserror 2.0.12",
|
||||
"time",
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-protos"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"flatbuffers 25.2.10",
|
||||
"prost",
|
||||
"rustfs-common",
|
||||
"tonic",
|
||||
"tonic-build",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-rio"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"aes-gcm",
|
||||
"async-trait",
|
||||
"base64-simd",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"crc32fast",
|
||||
"criterion",
|
||||
"futures",
|
||||
"hex-simd",
|
||||
"http 1.3.1",
|
||||
"md-5",
|
||||
"pin-project-lite",
|
||||
@@ -8426,6 +8254,48 @@ dependencies = [
|
||||
"urlencoding",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-api"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"datafusion",
|
||||
"futures",
|
||||
"futures-core",
|
||||
"http 1.3.1",
|
||||
"object_store",
|
||||
"pin-project-lite",
|
||||
"rustfs-common",
|
||||
"rustfs-ecstore",
|
||||
"s3s",
|
||||
"snafu",
|
||||
"tokio",
|
||||
"tokio-util",
|
||||
"tracing",
|
||||
"transform-stream",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-s3select-query"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"async-recursion",
|
||||
"async-trait",
|
||||
"datafusion",
|
||||
"derive_builder",
|
||||
"futures",
|
||||
"lazy_static",
|
||||
"parking_lot",
|
||||
"rustfs-s3select-api",
|
||||
"s3s",
|
||||
"snafu",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-signer"
|
||||
version = "0.0.1"
|
||||
@@ -8434,11 +8304,8 @@ dependencies = [
|
||||
"http 1.3.1",
|
||||
"hyper 1.6.0",
|
||||
"lazy_static",
|
||||
"rand 0.9.1",
|
||||
"rustfs-utils",
|
||||
"serde",
|
||||
"serde_urlencoded",
|
||||
"tempfile",
|
||||
"time",
|
||||
"tracing",
|
||||
]
|
||||
@@ -8451,7 +8318,6 @@ dependencies = [
|
||||
"blake3",
|
||||
"brotli 8.0.1",
|
||||
"bytes",
|
||||
"common",
|
||||
"crc32fast",
|
||||
"flate2",
|
||||
"futures",
|
||||
@@ -8488,17 +8354,22 @@ dependencies = [
|
||||
"zstd",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-workers"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustfs-zip"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"async-compression",
|
||||
"async_zip",
|
||||
"tokio",
|
||||
"tokio-stream",
|
||||
"tokio-tar",
|
||||
"xz2",
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11490,15 +11361,6 @@ dependencies = [
|
||||
"bitflags 2.9.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "workers"
|
||||
version = "0.0.1"
|
||||
dependencies = [
|
||||
"common",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "wrapcenum-derive"
|
||||
version = "0.4.1"
|
||||
|
||||
66
Cargo.toml
66
Cargo.toml
@@ -14,28 +14,28 @@
|
||||
|
||||
[workspace]
|
||||
members = [
|
||||
"appauth", # Application authentication and authorization
|
||||
"rustfs", # Core file system implementation
|
||||
"cli/rustfs-gui", # Graphical user interface client
|
||||
"common/common", # Shared utilities and data structures
|
||||
"crates/filemeta", # File metadata management
|
||||
"common/lock", # Distributed locking implementation
|
||||
"common/protos", # Protocol buffer definitions
|
||||
"common/workers", # Worker thread pools and task scheduling
|
||||
"crates/appauth", # Application authentication and authorization
|
||||
"crates/common", # Shared utilities and data structures
|
||||
"crates/config", # Configuration management
|
||||
"crates/crypto", # Cryptography and security features
|
||||
"crates/ecstore", # Erasure coding storage implementation
|
||||
"crates/e2e_test", # End-to-end test suite
|
||||
"crates/filemeta", # File metadata management
|
||||
"crates/iam", # Identity and Access Management
|
||||
"crates/lock", # Distributed locking implementation
|
||||
"crates/madmin", # Management dashboard and admin API interface
|
||||
"crates/notify", # Notification system for events
|
||||
"crates/obs", # Observability utilities
|
||||
"crates/protos", # Protocol buffer definitions
|
||||
"crates/rio", # Rust I/O utilities and abstractions
|
||||
"crates/utils", # Utility functions and helpers
|
||||
"crates/zip", # ZIP file handling and compression
|
||||
"crates/s3select-api", # S3 Select API interface
|
||||
"crates/s3select-query", # S3 Select query engine
|
||||
"crates/signer", # client signer
|
||||
"crypto", # Cryptography and security features
|
||||
"ecstore", # Erasure coding storage implementation
|
||||
"e2e_test", # End-to-end test suite
|
||||
"iam", # Identity and Access Management
|
||||
"madmin", # Management dashboard and admin API interface
|
||||
"rustfs", # Core file system implementation
|
||||
"s3select/api", # S3 Select API interface
|
||||
"s3select/query", # S3 Select query engine
|
||||
"crates/utils", # Utility functions and helpers
|
||||
"crates/workers", # Worker thread pools and task scheduling
|
||||
"crates/zip", # ZIP file handling and compression
|
||||
]
|
||||
resolver = "2"
|
||||
|
||||
@@ -53,17 +53,17 @@ unsafe_code = "deny"
|
||||
all = "warn"
|
||||
|
||||
[workspace.dependencies]
|
||||
api = { path = "./s3select/api", version = "0.0.1" }
|
||||
appauth = { path = "./appauth", version = "0.0.1" }
|
||||
common = { path = "./common/common", version = "0.0.1" }
|
||||
crypto = { path = "./crypto", version = "0.0.1" }
|
||||
ecstore = { path = "./ecstore", version = "0.0.1" }
|
||||
iam = { path = "./iam", version = "0.0.1" }
|
||||
lock = { path = "./common/lock", version = "0.0.1" }
|
||||
madmin = { path = "./madmin", version = "0.0.1" }
|
||||
policy = { path = "./policy", version = "0.0.1" }
|
||||
protos = { path = "./common/protos", version = "0.0.1" }
|
||||
query = { path = "./s3select/query", version = "0.0.1" }
|
||||
rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.1" }
|
||||
rustfs-appauth = { path = "crates/appauth", version = "0.0.1" }
|
||||
rustfs-common = { path = "crates/common", version = "0.0.1" }
|
||||
rustfs-crypto = { path = "crates/crypto", version = "0.0.1" }
|
||||
rustfs-ecstore = { path = "crates/ecstore", version = "0.0.1" }
|
||||
rustfs-iam = { path = "crates/iam", version = "0.0.1" }
|
||||
rustfs-lock = { path = "crates/lock", version = "0.0.1" }
|
||||
rustfs-madmin = { path = "crates/madmin", version = "0.0.1" }
|
||||
rustfs-policy = { path = "crates/policy", version = "0.0.1" }
|
||||
rustfs-protos = { path = "crates/protos", version = "0.0.1" }
|
||||
rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.1" }
|
||||
rustfs = { path = "./rustfs", version = "0.0.1" }
|
||||
rustfs-zip = { path = "./crates/zip", version = "0.0.1" }
|
||||
rustfs-config = { path = "./crates/config", version = "0.0.1" }
|
||||
@@ -73,7 +73,7 @@ rustfs-utils = { path = "crates/utils", version = "0.0.1" }
|
||||
rustfs-rio = { path = "crates/rio", version = "0.0.1" }
|
||||
rustfs-filemeta = { path = "crates/filemeta", version = "0.0.1" }
|
||||
rustfs-signer = { path = "crates/signer", version = "0.0.1" }
|
||||
workers = { path = "./common/workers", version = "0.0.1" }
|
||||
rustfs-workers = { path = "crates/workers", version = "0.0.1" }
|
||||
aes-gcm = { version = "0.10.3", features = ["std"] }
|
||||
arc-swap = "1.7.1"
|
||||
argon2 = { version = "0.5.3", features = ["std"] }
|
||||
@@ -81,15 +81,14 @@ atoi = "2.0.0"
|
||||
async-channel = "2.3.1"
|
||||
async-recursion = "1.1.1"
|
||||
async-trait = "0.1.88"
|
||||
async-compression = { version = "0.4.0" }
|
||||
atomic_enum = "0.3.0"
|
||||
aws-sdk-s3 = "1.95.0"
|
||||
axum = "0.8.4"
|
||||
axum-extra = "0.10.1"
|
||||
axum-server = { version = "0.7.2", features = ["tls-rustls"] }
|
||||
backon = "1.5.1"
|
||||
base64-simd = "0.8.0"
|
||||
base64 = "0.22.1"
|
||||
blake2 = "0.10.6"
|
||||
brotli = "8.0.1"
|
||||
bytes = { version = "1.10.1", features = ["serde"] }
|
||||
bytesize = "2.0.1"
|
||||
@@ -100,6 +99,7 @@ chrono = { version = "0.4.41", features = ["serde"] }
|
||||
clap = { version = "4.5.40", features = ["derive", "env"] }
|
||||
const-str = { version = "0.6.2", features = ["std", "proc"] }
|
||||
crc32fast = "1.4.2"
|
||||
criterion = { version = "0.5", features = ["html_reports"] }
|
||||
dashmap = "6.1.0"
|
||||
datafusion = "46.0.1"
|
||||
derive_builder = "0.20.2"
|
||||
@@ -128,9 +128,7 @@ hyper-rustls = "0.27.5"
|
||||
http = "1.3.1"
|
||||
http-body = "1.0.1"
|
||||
humantime = "2.2.0"
|
||||
include_dir = "0.7.4"
|
||||
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
||||
itertools = "0.14.0"
|
||||
jsonwebtoken = "9.3.1"
|
||||
keyring = { version = "3.6.2", features = [
|
||||
"apple-native",
|
||||
@@ -143,7 +141,6 @@ local-ip-address = "0.6.5"
|
||||
lz4 = "1.28.1"
|
||||
matchit = "0.8.4"
|
||||
md-5 = "0.10.6"
|
||||
mime = "0.3.17"
|
||||
mime_guess = "2.0.5"
|
||||
netif = "0.1.6"
|
||||
nix = { version = "0.30.1", features = ["fs"] }
|
||||
@@ -175,7 +172,6 @@ percent-encoding = "2.3.1"
|
||||
pin-project-lite = "0.2.16"
|
||||
prost = "0.13.5"
|
||||
prost-build = "0.13.5"
|
||||
protobuf = "3.7"
|
||||
quick-xml = "0.37.5"
|
||||
rand = "0.9.1"
|
||||
rdkafka = { version = "0.37.0", features = ["tokio"] }
|
||||
@@ -205,7 +201,6 @@ rustls = { version = "0.23.28" }
|
||||
rustls-pki-types = "1.12.0"
|
||||
rustls-pemfile = "2.2.0"
|
||||
s3s = { git = "https://github.com/Nugine/s3s.git", rev = "4733cdfb27b2713e832967232cbff413bb768c10" }
|
||||
scopeguard = "1.2.0"
|
||||
shadow-rs = { version = "1.2.0", default-features = false }
|
||||
serde = { version = "1.0.219", features = ["derive"] }
|
||||
serde_json = { version = "1.0.140", features = ["raw_value"] }
|
||||
@@ -213,7 +208,6 @@ serde-xml-rs = "0.8.1"
|
||||
serde_urlencoded = "0.7.1"
|
||||
sha1 = "0.10.6"
|
||||
sha2 = "0.10.9"
|
||||
std-next = "0.1.9"
|
||||
siphasher = "1.0.1"
|
||||
smallvec = { version = "1.15.1", features = ["serde"] }
|
||||
snafu = "0.8.6"
|
||||
|
||||
@@ -1,360 +0,0 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use tracing_error::{SpanTrace, SpanTraceStatus};
|
||||
|
||||
pub type StdError = Box<dyn std::error::Error + Send + Sync + 'static>;
|
||||
|
||||
pub type Result<T = (), E = Error> = std::result::Result<T, E>;
|
||||
|
||||
#[derive(Debug)]
|
||||
pub struct Error {
|
||||
inner: Box<dyn std::error::Error + Send + Sync + 'static>,
|
||||
span_trace: SpanTrace,
|
||||
}
|
||||
|
||||
impl Error {
|
||||
pub fn other<E>(error: E) -> Self
|
||||
where
|
||||
E: std::fmt::Display + Into<Box<dyn std::error::Error + Send + Sync>>,
|
||||
{
|
||||
Self::from_std_error(error.into())
|
||||
}
|
||||
|
||||
/// Create a new error from a `std::error::Error`.
|
||||
#[must_use]
|
||||
#[track_caller]
|
||||
pub fn new<T: std::error::Error + Send + Sync + 'static>(source: T) -> Self {
|
||||
Self::from_std_error(source.into())
|
||||
}
|
||||
|
||||
/// Create a new error from a `std::error::Error`.
|
||||
#[must_use]
|
||||
#[track_caller]
|
||||
pub fn from_std_error(inner: StdError) -> Self {
|
||||
Self {
|
||||
inner,
|
||||
span_trace: SpanTrace::capture(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new error from a string.
|
||||
#[must_use]
|
||||
#[track_caller]
|
||||
pub fn from_string(s: impl Into<String>) -> Self {
|
||||
Self::msg(s)
|
||||
}
|
||||
|
||||
/// Create a new error from a string.
|
||||
#[must_use]
|
||||
#[track_caller]
|
||||
pub fn msg(s: impl Into<String>) -> Self {
|
||||
Self::from_std_error(s.into().into())
|
||||
}
|
||||
|
||||
/// Returns `true` if the inner type is the same as `T`.
|
||||
#[inline]
|
||||
pub fn is<T: std::error::Error + 'static>(&self) -> bool {
|
||||
self.inner.is::<T>()
|
||||
}
|
||||
|
||||
/// Returns some reference to the inner value if it is of type `T`, or
|
||||
/// `None` if it isn't.
|
||||
#[inline]
|
||||
pub fn downcast_ref<T: std::error::Error + 'static>(&self) -> Option<&T> {
|
||||
self.inner.downcast_ref()
|
||||
}
|
||||
|
||||
/// Returns some mutable reference to the inner value if it is of type `T`, or
|
||||
/// `None` if it isn't.
|
||||
#[inline]
|
||||
pub fn downcast_mut<T: std::error::Error + 'static>(&mut self) -> Option<&mut T> {
|
||||
self.inner.downcast_mut()
|
||||
}
|
||||
|
||||
pub fn to_io_err(&self) -> Option<std::io::Error> {
|
||||
self.downcast_ref::<std::io::Error>()
|
||||
.map(|e| std::io::Error::new(e.kind(), e.to_string()))
|
||||
}
|
||||
|
||||
pub fn inner_string(&self) -> String {
|
||||
self.inner.to_string()
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: std::error::Error + Send + Sync + 'static> From<T> for Error {
|
||||
fn from(e: T) -> Self {
|
||||
Self::new(e)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Display for Error {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "{}", self.inner)?;
|
||||
|
||||
if self.span_trace.status() != SpanTraceStatus::EMPTY {
|
||||
write!(f, "\nspan_trace:\n{}", self.span_trace)?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io;
|
||||
|
||||
#[derive(Debug)]
|
||||
struct CustomTestError {
|
||||
message: String,
|
||||
}
|
||||
|
||||
impl std::fmt::Display for CustomTestError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "Custom test error: {}", self.message)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for CustomTestError {}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct AnotherTestError;
|
||||
|
||||
impl std::fmt::Display for AnotherTestError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
write!(f, "Another test error")
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for AnotherTestError {}
|
||||
|
||||
#[test]
|
||||
fn test_error_new_from_std_error() {
|
||||
let io_error = io::Error::new(io::ErrorKind::NotFound, "File not found");
|
||||
let error = Error::new(io_error);
|
||||
|
||||
assert!(error.inner_string().contains("File not found"));
|
||||
assert!(error.is::<io::Error>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_from_std_error() {
|
||||
let io_error = io::Error::new(io::ErrorKind::PermissionDenied, "Permission denied");
|
||||
let boxed_error: StdError = Box::new(io_error);
|
||||
let error = Error::from_std_error(boxed_error);
|
||||
|
||||
assert!(error.inner_string().contains("Permission denied"));
|
||||
assert!(error.is::<io::Error>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_from_string() {
|
||||
let error = Error::from_string("Test error message");
|
||||
assert_eq!(error.inner_string(), "Test error message");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_msg() {
|
||||
let error = Error::msg("Another test message");
|
||||
assert_eq!(error.inner_string(), "Another test message");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_msg_with_string() {
|
||||
let message = String::from("String message");
|
||||
let error = Error::msg(message);
|
||||
assert_eq!(error.inner_string(), "String message");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_is_type_checking() {
|
||||
let io_error = io::Error::new(io::ErrorKind::InvalidInput, "Invalid input");
|
||||
let error = Error::new(io_error);
|
||||
|
||||
assert!(error.is::<io::Error>());
|
||||
assert!(!error.is::<CustomTestError>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_downcast_ref() {
|
||||
let io_error = io::Error::new(io::ErrorKind::TimedOut, "Operation timed out");
|
||||
let error = Error::new(io_error);
|
||||
|
||||
let downcast_io = error.downcast_ref::<io::Error>();
|
||||
assert!(downcast_io.is_some());
|
||||
assert_eq!(downcast_io.unwrap().kind(), io::ErrorKind::TimedOut);
|
||||
|
||||
let downcast_custom = error.downcast_ref::<CustomTestError>();
|
||||
assert!(downcast_custom.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_downcast_mut() {
|
||||
let io_error = io::Error::new(io::ErrorKind::Interrupted, "Operation interrupted");
|
||||
let mut error = Error::new(io_error);
|
||||
|
||||
let downcast_io = error.downcast_mut::<io::Error>();
|
||||
assert!(downcast_io.is_some());
|
||||
assert_eq!(downcast_io.unwrap().kind(), io::ErrorKind::Interrupted);
|
||||
|
||||
let downcast_custom = error.downcast_mut::<CustomTestError>();
|
||||
assert!(downcast_custom.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_to_io_err() {
|
||||
// Test with IO error
|
||||
let original_io_error = io::Error::new(io::ErrorKind::BrokenPipe, "Broken pipe");
|
||||
let error = Error::new(original_io_error);
|
||||
|
||||
let converted_io_error = error.to_io_err();
|
||||
assert!(converted_io_error.is_some());
|
||||
let io_err = converted_io_error.unwrap();
|
||||
assert_eq!(io_err.kind(), io::ErrorKind::BrokenPipe);
|
||||
assert!(io_err.to_string().contains("Broken pipe"));
|
||||
|
||||
// Test with non-IO error
|
||||
let custom_error = CustomTestError {
|
||||
message: "Not an IO error".to_string(),
|
||||
};
|
||||
let error = Error::new(custom_error);
|
||||
|
||||
let converted_io_error = error.to_io_err();
|
||||
assert!(converted_io_error.is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_inner_string() {
|
||||
let custom_error = CustomTestError {
|
||||
message: "Test message".to_string(),
|
||||
};
|
||||
let error = Error::new(custom_error);
|
||||
|
||||
assert_eq!(error.inner_string(), "Custom test error: Test message");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_from_trait() {
|
||||
let io_error = io::Error::new(io::ErrorKind::UnexpectedEof, "Unexpected EOF");
|
||||
let error: Error = io_error.into();
|
||||
|
||||
assert!(error.is::<io::Error>());
|
||||
assert!(error.inner_string().contains("Unexpected EOF"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_display() {
|
||||
let custom_error = CustomTestError {
|
||||
message: "Display test".to_string(),
|
||||
};
|
||||
let error = Error::new(custom_error);
|
||||
|
||||
let display_string = format!("{}", error);
|
||||
assert!(display_string.contains("Custom test error: Display test"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_debug() {
|
||||
let error = Error::msg("Debug test");
|
||||
let debug_string = format!("{:?}", error);
|
||||
|
||||
assert!(debug_string.contains("Error"));
|
||||
assert!(debug_string.contains("inner"));
|
||||
assert!(debug_string.contains("span_trace"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_error_types() {
|
||||
let errors = vec![
|
||||
Error::new(io::Error::new(io::ErrorKind::NotFound, "Not found")),
|
||||
Error::new(CustomTestError {
|
||||
message: "Custom".to_string(),
|
||||
}),
|
||||
Error::new(AnotherTestError),
|
||||
Error::msg("String error"),
|
||||
];
|
||||
|
||||
assert!(errors[0].is::<io::Error>());
|
||||
assert!(errors[1].is::<CustomTestError>());
|
||||
assert!(errors[2].is::<AnotherTestError>());
|
||||
assert!(!errors[3].is::<io::Error>());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_chain_compatibility() {
|
||||
// Test that our Error type works well with error chains
|
||||
let io_error = io::Error::new(io::ErrorKind::InvalidData, "Invalid data");
|
||||
let error = Error::new(io_error);
|
||||
|
||||
// Should be able to convert back to Result
|
||||
let result: Result<(), Error> = Err(error);
|
||||
assert!(result.is_err());
|
||||
|
||||
// Test the error from the result
|
||||
if let Err(err) = result {
|
||||
assert!(err.is::<io::Error>());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_result_type_alias() {
|
||||
// Test the Result type alias
|
||||
fn test_function() -> Result<String> {
|
||||
Ok("Success".to_string())
|
||||
}
|
||||
|
||||
fn test_function_with_error() -> Result<String> {
|
||||
Err(Error::msg("Test error"))
|
||||
}
|
||||
|
||||
let success_result = test_function();
|
||||
assert!(success_result.is_ok());
|
||||
assert_eq!(success_result.unwrap(), "Success");
|
||||
|
||||
let error_result = test_function_with_error();
|
||||
assert!(error_result.is_err());
|
||||
assert_eq!(error_result.unwrap_err().inner_string(), "Test error");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_with_empty_message() {
|
||||
let error = Error::msg("");
|
||||
assert_eq!(error.inner_string(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_with_unicode_message() {
|
||||
let unicode_message = "错误信息 🚨 Error message with émojis and ñon-ASCII";
|
||||
let error = Error::msg(unicode_message);
|
||||
assert_eq!(error.inner_string(), unicode_message);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_error_with_very_long_message() {
|
||||
let long_message = "A".repeat(10000);
|
||||
let error = Error::msg(&long_message);
|
||||
assert_eq!(error.inner_string(), long_message);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_span_trace_capture() {
|
||||
// Test that span trace is captured (though we can't easily test the content)
|
||||
let error = Error::msg("Span trace test");
|
||||
let display_string = format!("{}", error);
|
||||
|
||||
// The error should at least contain the message
|
||||
assert!(display_string.contains("Span trace test"));
|
||||
}
|
||||
}
|
||||
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "appauth"
|
||||
name = "rustfs-appauth"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
@@ -22,9 +22,6 @@ version.workspace = true
|
||||
|
||||
[dependencies]
|
||||
base64-simd = { workspace = true }
|
||||
common.workspace = true
|
||||
hex-simd = { workspace = true }
|
||||
rand.workspace = true
|
||||
rsa = { workspace = true }
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
@@ -13,17 +13,17 @@
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "common"
|
||||
name = "rustfs-common"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
async-trait.workspace = true
|
||||
lazy_static.workspace = true
|
||||
scopeguard = { workspace = true }
|
||||
tokio.workspace = true
|
||||
tonic = { workspace = true }
|
||||
tracing-error.workspace = true
|
||||
@@ -16,18 +16,18 @@ use std::collections::HashMap;
|
||||
|
||||
use crate::last_minute::{self};
|
||||
pub struct ReplicationLatency {
|
||||
// 单个和多部分 PUT 请求的延迟
|
||||
// Delays for single and multipart PUT requests
|
||||
upload_histogram: last_minute::LastMinuteHistogram,
|
||||
}
|
||||
|
||||
impl ReplicationLatency {
|
||||
// 合并两个 ReplicationLatency
|
||||
// Merge two ReplicationLatency
|
||||
pub fn merge(&mut self, other: &mut ReplicationLatency) -> &ReplicationLatency {
|
||||
self.upload_histogram.merge(&other.upload_histogram);
|
||||
self
|
||||
}
|
||||
|
||||
// 获取上传延迟（按对象大小区间分类）
|
||||
// Get upload delay (categorized by object size interval)
|
||||
pub fn get_upload_latency(&mut self) -> HashMap<String, u64> {
|
||||
let mut ret = HashMap::new();
|
||||
let avg = self.upload_histogram.get_avg_data();
|
||||
@@ -41,7 +41,7 @@ impl ReplicationLatency {
|
||||
self.upload_histogram.add(size, during);
|
||||
}
|
||||
|
||||
// 模拟从 size tag 到字符串的转换
|
||||
// Simulate the conversion from size tag to string
|
||||
fn size_tag_to_string(&self, tag: usize) -> String {
|
||||
match tag {
|
||||
0 => String::from("Size < 1 KiB"),
|
||||
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "crypto"
|
||||
name = "rustfs-crypto"
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
repository.workspace = true
|
||||
@@ -20,27 +20,21 @@ license.workspace = true
|
||||
repository.workspace = true
|
||||
rust-version.workspace = true
|
||||
|
||||
|
||||
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
ecstore.workspace = true
|
||||
rustfs-ecstore.workspace = true
|
||||
flatbuffers.workspace = true
|
||||
futures.workspace = true
|
||||
lazy_static.workspace = true
|
||||
lock.workspace = true
|
||||
protos.workspace = true
|
||||
rustfs-lock.workspace = true
|
||||
rustfs-protos.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
tonic = { workspace = true }
|
||||
tokio = { workspace = true }
|
||||
tower.workspace = true
|
||||
url.workspace = true
|
||||
madmin.workspace =true
|
||||
common.workspace = true
|
||||
rustfs-madmin.workspace = true
|
||||
rustfs-filemeta.workspace = true
|
||||
bytes.workspace = true
|
||||
@@ -13,15 +13,14 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use std::{error::Error, sync::Arc, time::Duration};
|
||||
|
||||
use lock::{
|
||||
use rustfs_lock::{
|
||||
drwmutex::Options,
|
||||
lock_args::LockArgs,
|
||||
namespace_lock::{NsLockMap, new_nslock},
|
||||
new_lock_api,
|
||||
};
|
||||
use protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest};
|
||||
use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest};
|
||||
use std::{error::Error, sync::Arc, time::Duration};
|
||||
use tokio::sync::RwLock;
|
||||
use tonic::Request;
|
||||
|
||||
@@ -13,21 +13,21 @@
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
use ecstore::disk::{VolumeInfo, WalkDirOptions};
|
||||
|
||||
use futures::future::join_all;
|
||||
use protos::proto_gen::node_service::WalkDirRequest;
|
||||
use protos::{
|
||||
use rmp_serde::{Deserializer, Serializer};
|
||||
use rustfs_ecstore::disk::{VolumeInfo, WalkDirOptions};
|
||||
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
|
||||
use rustfs_protos::proto_gen::node_service::WalkDirRequest;
|
||||
use rustfs_protos::{
|
||||
models::{PingBody, PingBodyBuilder},
|
||||
node_service_time_out_client,
|
||||
proto_gen::node_service::{
|
||||
ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, PingResponse, ReadAllRequest,
|
||||
},
|
||||
};
|
||||
use rmp_serde::{Deserializer, Serializer};
|
||||
use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{error::Error, io::Cursor};
|
||||
use std::error::Error;
|
||||
use std::io::Cursor;
|
||||
use tokio::spawn;
|
||||
use tonic::Request;
|
||||
use tonic::codegen::tokio_stream::StreamExt;
|
||||
@@ -199,7 +199,7 @@ async fn storage_info() -> Result<(), Box<dyn Error>> {
|
||||
let info = response.storage_info;
|
||||
|
||||
let mut buf = Deserializer::new(Cursor::new(info));
|
||||
let storage_info: madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
|
||||
let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap();
|
||||
println!("{storage_info:?}");
|
||||
Ok(())
|
||||
}
|
||||
@@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
|
||||
[package]
|
||||
name = "ecstore"
|
||||
name = "rustfs-ecstore"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
@@ -32,8 +32,8 @@ rustfs-config = { workspace = true, features = ["constants", "notify"] }
|
||||
async-trait.workspace = true
|
||||
bytes.workspace = true
|
||||
byteorder = { workspace = true }
|
||||
common.workspace = true
|
||||
policy.workspace = true
|
||||
rustfs-common.workspace = true
|
||||
rustfs-policy.workspace = true
|
||||
chrono.workspace = true
|
||||
glob = { workspace = true }
|
||||
thiserror.workspace = true
|
||||
@@ -51,10 +51,10 @@ url.workspace = true
|
||||
uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] }
|
||||
reed-solomon-simd = { workspace = true }
|
||||
lazy_static.workspace = true
|
||||
lock.workspace = true
|
||||
rustfs-lock.workspace = true
|
||||
regex = { workspace = true }
|
||||
path-absolutize = { workspace = true }
|
||||
protos.workspace = true
|
||||
rustfs-protos.workspace = true
|
||||
rmp.workspace = true
|
||||
rmp-serde.workspace = true
|
||||
tokio-util = { workspace = true, features = ["io", "compat"] }
|
||||
@@ -79,8 +79,8 @@ num_cpus = { workspace = true }
|
||||
rand.workspace = true
|
||||
pin-project-lite.workspace = true
|
||||
md-5.workspace = true
|
||||
madmin.workspace = true
|
||||
workers.workspace = true
|
||||
rustfs-madmin.workspace = true
|
||||
rustfs-workers.workspace = true
|
||||
reqwest = { workspace = true }
|
||||
aws-sdk-s3 = { workspace = true }
|
||||
once_cell = { workspace = true }
|
||||
@@ -1,6 +1,7 @@
|
||||
# ECStore - Erasure Coding Storage
|
||||
|
||||
ECStore provides erasure coding functionality for the RustFS project, using high-performance Reed-Solomon SIMD implementation for optimal performance.
|
||||
ECStore provides erasure coding functionality for the RustFS project, using high-performance Reed-Solomon SIMD
|
||||
implementation for optimal performance.
|
||||
|
||||
## Reed-Solomon Implementation
|
||||
|
||||
@@ -14,7 +15,7 @@ ECStore provides erasure coding functionality for the RustFS project, using high
|
||||
### Usage Example
|
||||
|
||||
```rust
|
||||
use ecstore::erasure_coding::Erasure;
|
||||
use rustfs_ecstore::erasure_coding::Erasure;
|
||||
|
||||
// Create erasure coding instance
|
||||
// 4 data shards, 2 parity shards, 1KB block size
|
||||
@@ -22,22 +23,22 @@ let erasure = Erasure::new(4, 2, 1024);
|
||||
|
||||
// Encode data
|
||||
let data = b"hello world from rustfs erasure coding";
|
||||
let shards = erasure.encode_data(data)?;
|
||||
let shards = erasure.encode_data(data) ?;
|
||||
|
||||
// Simulate loss of one shard
|
||||
let mut shards_opt: Vec<Option<Vec<u8>>> = shards
|
||||
.iter()
|
||||
.map(|b| Some(b.to_vec()))
|
||||
.collect();
|
||||
let mut shards_opt: Vec<Option<Vec<u8> > > = shards
|
||||
.iter()
|
||||
.map( | b| Some(b.to_vec()))
|
||||
.collect();
|
||||
shards_opt[2] = None; // Lose shard 2
|
||||
|
||||
// Reconstruct missing data
|
||||
erasure.decode_data(&mut shards_opt)?;
|
||||
erasure.decode_data( & mut shards_opt) ?;
|
||||
|
||||
// Recover original data
|
||||
let mut recovered = Vec::new();
|
||||
for shard in shards_opt.iter().take(4) { // Only data shards
|
||||
recovered.extend_from_slice(shard.as_ref().unwrap());
|
||||
recovered.extend_from_slice(shard.as_ref().unwrap());
|
||||
}
|
||||
recovered.truncate(data.len());
|
||||
assert_eq!(&recovered, data);
|
||||
@@ -46,6 +47,7 @@ assert_eq!(&recovered, data);
|
||||
## Performance Considerations
|
||||
|
||||
### SIMD Implementation Benefits
|
||||
|
||||
- **High Throughput**: Optimized for large block sizes (>= 1KB recommended)
|
||||
- **CPU Optimization**: Leverages modern CPU SIMD instructions
|
||||
- **Scalability**: Excellent performance for high-throughput scenarios
|
||||
@@ -53,6 +55,7 @@ assert_eq!(&recovered, data);
|
||||
### Implementation Details
|
||||
|
||||
#### `reed-solomon-simd`
|
||||
|
||||
- **Instance Caching**: Encoder/decoder instances are cached and reused for optimal performance
|
||||
- **Thread Safety**: Thread-safe with RwLock-based caching
|
||||
- **SIMD Optimization**: Leverages CPU SIMD instructions for maximum performance
|
||||
@@ -68,15 +71,18 @@ assert_eq!(&recovered, data);
|
||||
## Cross-Platform Compatibility
|
||||
|
||||
The SIMD implementation supports:
|
||||
|
||||
- x86_64 with advanced SIMD instructions (AVX2, SSE)
|
||||
- aarch64 (ARM64) with NEON SIMD optimizations
|
||||
- Other architectures with fallback implementations
|
||||
|
||||
The implementation automatically selects the best available SIMD instructions for the target platform, providing optimal performance across different architectures.
|
||||
The implementation automatically selects the best available SIMD instructions for the target platform, providing optimal
|
||||
performance across different architectures.
|
||||
|
||||
## Testing and Benchmarking
|
||||
|
||||
Run performance benchmarks:
|
||||
|
||||
```bash
|
||||
# Run erasure coding benchmarks
|
||||
cargo bench --bench erasure_benchmark
|
||||
@@ -91,6 +97,7 @@ cargo bench --bench comparison_benchmark
|
||||
## Error Handling
|
||||
|
||||
All operations return `Result` types with comprehensive error information:
|
||||
|
||||
- Encoding errors: Invalid parameters, insufficient memory
|
||||
- Decoding errors: Too many missing shards, corrupted data
|
||||
- Configuration errors: Invalid shard counts, unsupported parameters
|
||||
@@ -33,7 +33,7 @@
|
||||
//! ```
|
||||
|
||||
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use ecstore::erasure_coding::Erasure;
|
||||
use rustfs_ecstore::erasure_coding::Erasure;
|
||||
use std::time::Duration;
|
||||
|
||||
/// Performance test data configuration
|
||||
@@ -44,7 +44,7 @@
|
||||
//! - SIMD optimization for different shard sizes
|
||||
|
||||
use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main};
|
||||
use ecstore::erasure_coding::{Erasure, calc_shard_size};
|
||||
use rustfs_ecstore::erasure_coding::{Erasure, calc_shard_size};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Benchmark configuration structure
|
||||
@@ -25,12 +25,14 @@ use crate::{
|
||||
notification_sys::get_global_notification_sys,
|
||||
store_api::StorageAPI,
|
||||
};
|
||||
use common::{
|
||||
use rustfs_common::{
|
||||
// error::{Error, Result},
|
||||
globals::GLOBAL_Local_Node_Name,
|
||||
};
|
||||
use madmin::{BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, ServerProperties};
|
||||
use protos::{
|
||||
use rustfs_madmin::{
|
||||
BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, ServerProperties,
|
||||
};
|
||||
use rustfs_protos::{
|
||||
models::{PingBody, PingBodyBuilder},
|
||||
node_service_time_out_client,
|
||||
proto_gen::node_service::{PingRequest, PingResponse},
|
||||
@@ -219,13 +221,13 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
warn!("server_info end {:?}", after2 - after1);
|
||||
servers.push(local);
|
||||
|
||||
let mut buckets = madmin::Buckets::default();
|
||||
let mut objects = madmin::Objects::default();
|
||||
let mut versions = madmin::Versions::default();
|
||||
let mut delete_markers = madmin::DeleteMarkers::default();
|
||||
let mut usage = madmin::Usage::default();
|
||||
let mut buckets = rustfs_madmin::Buckets::default();
|
||||
let mut objects = rustfs_madmin::Objects::default();
|
||||
let mut versions = rustfs_madmin::Versions::default();
|
||||
let mut delete_markers = rustfs_madmin::DeleteMarkers::default();
|
||||
let mut usage = rustfs_madmin::Usage::default();
|
||||
let mut mode = ITEM_INITIALIZING;
|
||||
let mut backend = madmin::ErasureBackend::default();
|
||||
let mut backend = rustfs_madmin::ErasureBackend::default();
|
||||
let mut pools: HashMap<i32, HashMap<i32, ErasureSetInfo>> = HashMap::new();
|
||||
|
||||
if let Some(store) = new_object_layer_fn() {
|
||||
@@ -266,8 +268,8 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
let after5 = OffsetDateTime::now_utc();
|
||||
|
||||
warn!("get_online_offline_disks_stats end {:?}", after5 - after4);
|
||||
backend = madmin::ErasureBackend {
|
||||
backend_type: madmin::BackendType::ErasureType,
|
||||
backend = rustfs_madmin::ErasureBackend {
|
||||
backend_type: rustfs_madmin::BackendType::ErasureType,
|
||||
online_disks: online_disks.sum(),
|
||||
offline_disks: offline_disks.sum(),
|
||||
standard_sc_parity: backen_info.standard_sc_parity,
|
||||
@@ -283,7 +285,7 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage {
|
||||
}
|
||||
}
|
||||
|
||||
let services = madmin::Services::default();
|
||||
let services = rustfs_madmin::Services::default();
|
||||
|
||||
InfoMessage {
|
||||
mode: Some(mode.to_string()),
|
||||
@@ -17,8 +17,8 @@ use super::{quota::BucketQuota, target::BucketTargets};
|
||||
use super::object_lock::ObjectLockApi;
|
||||
use super::versioning::VersioningApi;
|
||||
use byteorder::{BigEndian, ByteOrder, LittleEndian};
|
||||
use policy::policy::BucketPolicy;
|
||||
use rmp_serde::Serializer as rmpSerializer;
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
|
||||
ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
|
||||
@@ -21,7 +21,7 @@ use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_la
|
||||
use crate::heal::heal_commands::HealOpts;
|
||||
use crate::store::ECStore;
|
||||
use futures::future::join_all;
|
||||
use policy::policy::BucketPolicy;
|
||||
use rustfs_policy::policy::BucketPolicy;
|
||||
use s3s::dto::{
|
||||
BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration,
|
||||
ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration,
|
||||
@@ -14,7 +14,7 @@
|
||||
|
||||
use super::{error::BucketMetadataError, metadata_sys::get_bucket_metadata_sys};
|
||||
use crate::error::Result;
|
||||
use policy::policy::{BucketPolicy, BucketPolicyArgs};
|
||||
use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs};
|
||||
use tracing::warn;
|
||||
|
||||
pub struct PolicySys {}
|
||||
@@ -18,7 +18,7 @@ use crate::bucket::versioning::VersioningApi;
|
||||
use crate::bucket::versioning_sys::BucketVersioningSys;
|
||||
use crate::store::ECStore;
|
||||
use crate::store_api::{ObjectOptions, ObjectToDelete};
|
||||
use lock::local_locker::MAX_DELETE_LIST;
|
||||
use rustfs_lock::local_locker::MAX_DELETE_LIST;
|
||||
|
||||
pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) {
|
||||
let mut remaining = to_del;
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user