diff --git a/Cargo.lock b/Cargo.lock index 330941e4..34f21265 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -173,43 +173,6 @@ version = "1.0.98" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" -[[package]] -name = "api" -version = "0.0.1" -dependencies = [ - "async-trait", - "bytes", - "chrono", - "common", - "datafusion", - "ecstore", - "futures", - "futures-core", - "http 1.3.1", - "object_store", - "pin-project-lite", - "s3s", - "snafu", - "tokio", - "tokio-util", - "tracing", - "transform-stream", - "url", -] - -[[package]] -name = "appauth" -version = "0.0.1" -dependencies = [ - "base64-simd", - "common", - "hex-simd", - "rand 0.9.1", - "rsa", - "serde", - "serde_json", -] - [[package]] name = "arbitrary" version = "1.4.1" @@ -653,20 +616,6 @@ dependencies = [ "syn 2.0.101", ] -[[package]] -name = "async_zip" -version = "0.0.17" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00b9f7252833d5ed4b00aa9604b563529dd5e11de9c23615de2dcdf91eb87b52" -dependencies = [ - "crc32fast", - "futures-lite", - "pin-project", - "thiserror 1.0.69", - "tokio", - "tokio-util", -] - [[package]] name = "atk" version = "0.18.2" @@ -1142,17 +1091,6 @@ dependencies = [ "tower-service", ] -[[package]] -name = "backon" -version = "1.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "302eaff5357a264a2c42f127ecb8bac761cf99749fc3dc95677e2743991f99e7" -dependencies = [ - "fastrand", - "gloo-timers", - "tokio", -] - [[package]] name = "backtrace" version = "0.3.75" @@ -1838,18 +1776,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "common" -version = "0.0.1" -dependencies = [ - "async-trait", - "lazy_static", - "scopeguard", - "tokio", - "tonic", - "tracing-error", -] - [[package]] name = "concurrent-queue" version = "2.5.0" @@ -2208,24 +2134,6 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" -[[package]] -name = "crypto" -version = "0.0.1" -dependencies = [ - "aes-gcm", - "argon2", - "cfg-if", - "chacha20poly1305", - "jsonwebtoken", - "pbkdf2", - "rand 0.9.1", - "serde_json", - "sha2 0.10.9", - "test-case", - "thiserror 2.0.12", - "time", -] - [[package]] name = "crypto-bigint" version = "0.4.9" @@ -3566,21 +3474,18 @@ name = "e2e_test" version = "0.0.1" dependencies = [ "bytes", - "common", - "ecstore", "flatbuffers 25.2.10", "futures", - "lazy_static", - "lock", - "madmin", - "protos", "rmp-serde", + "rustfs-ecstore", "rustfs-filemeta", + "rustfs-lock", + "rustfs-madmin", + "rustfs-protos", "serde", "serde_json", "tokio", "tonic", - "tower", "url", ] @@ -3596,81 +3501,6 @@ dependencies = [ "signature 1.6.4", ] -[[package]] -name = "ecstore" -version = "0.0.1" -dependencies = [ - "async-channel", - "async-trait", - "aws-sdk-s3", - "base64 0.22.1", - "byteorder", - "bytes", - "bytesize", - "chrono", - "common", - "criterion", - "enumset", - "flatbuffers 25.2.10", - "futures", - "futures-util", - "glob", - "hex-simd", - "hmac 0.12.1", - "http 1.3.1", - "hyper 1.6.0", - "hyper-rustls 0.27.7", - "hyper-util", - "lazy_static", - "lock", - "madmin", - "md-5", - "nix 0.30.1", - "num_cpus", - "once_cell", - "path-absolutize", - "path-clean", - "pin-project-lite", - "policy", - "protos", - "rand 0.9.1", - "reed-solomon-simd", - "regex", - "reqwest", - "rmp", - "rmp-serde", - "rustfs-config", - "rustfs-filemeta", - "rustfs-rio", - "rustfs-rsc", - "rustfs-signer", - "rustfs-utils", - "rustls 0.23.28", - "s3s", - "serde", - "serde-xml-rs 0.8.1", - "serde_json", - "sha2 0.10.9", - "shadow-rs", - "smallvec", - "temp-env", - "tempfile", - "thiserror 2.0.12", - "time", - "tokio", - "tokio-stream", - "tokio-util", - "tonic", - "tower", - "tracing", - "url", - "urlencoding", - "uuid", - "winapi", - "workers", - "xxhash-rust", -] - [[package]] name = "either" version = "1.15.0" @@ -4446,18 +4276,6 @@ dependencies = [ "web-sys", ] -[[package]] -name = "gloo-timers" -version = "0.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", -] - [[package]] name = "gloo-utils" version = "0.2.0" @@ -4904,36 +4722,6 @@ dependencies = [ "windows-registry", ] -[[package]] -name = "iam" -version = "0.0.1" -dependencies = [ - "arc-swap", - "async-trait", - "base64-simd", - "common", - "crypto", - "ecstore", - "futures", - "ipnetwork", - "itertools 0.14.0", - "jsonwebtoken", - "lazy_static", - "madmin", - "policy", - "rand 0.9.1", - "regex", - "rustfs-utils", - "serde", - "serde_json", - "strum", - "test-case", - "thiserror 2.0.12", - "time", - "tokio", - "tracing", -] - [[package]] name = "iana-time-zone" version = "0.1.63" @@ -5087,25 +4875,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "include_dir" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "923d117408f1e49d914f1a379a309cffe4f18c05cf4e3d12e613a15fc81bd0dd" -dependencies = [ - "include_dir_macros", -] - -[[package]] -name = "include_dir_macros" -version = "0.7.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cab85a7ed0bd5f0e76d93846e0147172bed2e2d3f859bcc33a8d9699cad1a75" -dependencies = [ - "proc-macro2", - "quote", -] - [[package]] name = "indexmap" version = "1.9.3" @@ -5662,26 +5431,6 @@ dependencies = [ "windows-sys 0.59.0", ] -[[package]] -name = "lock" -version = "0.0.1" -dependencies = [ - "async-trait", - "backon", - "common", - "lazy_static", - "protos", - "rand 0.9.1", - "serde", - "serde_json", - "tokio", - "tonic", - "tracing", - "tracing-error", - "url", - "uuid", -] - [[package]] name = "lock_api" version = "0.4.13" @@ -5778,21 +5527,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" -[[package]] -name = "madmin" -version = "0.0.1" -dependencies = [ - "chrono", - "common", - "humantime", - "hyper 1.6.0", - "s3s", - "serde", - "serde_json", - "time", - "tracing", -] - [[package]] name = "malloc_buf" version = "0.0.6" @@ -7202,33 +6936,6 @@ dependencies = [ "miniz_oxide", ] -[[package]] -name = "policy" -version = "0.0.1" -dependencies = [ - "arc-swap", - "async-trait", - "base64-simd", - "common", - "crypto", - "futures", - "ipnetwork", - "itertools 0.14.0", - "jsonwebtoken", - "lazy_static", - "madmin", - "rand 0.9.1", - "regex", - "serde", - "serde_json", - "strum", - "test-case", - "thiserror 2.0.12", - "time", - "tokio", - "tracing", -] - [[package]] name = "polling" version = "3.8.0" @@ -7450,41 +7157,6 @@ dependencies = [ "prost", ] -[[package]] -name = "protobuf" -version = "3.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" -dependencies = [ - "once_cell", - "protobuf-support", - "thiserror 1.0.69", -] - -[[package]] -name = "protobuf-support" -version = "3.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" -dependencies = [ - "thiserror 1.0.69", -] - -[[package]] -name = "protos" -version = "0.0.1" -dependencies = [ - "common", - "flatbuffers 25.2.10", - "prost", - "prost-build", - "protobuf", - "tokio", - "tonic", - "tonic-build", - "tower", -] - [[package]] name = "psm" version = "0.1.26" @@ -7494,24 +7166,6 @@ dependencies = [ "cc", ] -[[package]] -name = "query" -version = "0.0.1" -dependencies = [ - "api", - "async-recursion", - "async-trait", - "datafusion", - "derive_builder", - "futures", - "lazy_static", - "parking_lot", - "s3s", - "snafu", - "tokio", - "tracing", -] - [[package]] name = "quick-xml" version = "0.37.5" @@ -8178,59 +7832,45 @@ dependencies = [ name = "rustfs" version = "0.0.1" dependencies = [ - "api", - "appauth", "async-trait", "atoi", "atomic_enum", - "aws-sdk-s3", "axum", "axum-extra", "axum-server", - "base64 0.22.1", "bytes", "chrono", "clap", - "common", "const-str", - "crypto", "datafusion", - "ecstore", - "flatbuffers 25.2.10", "futures", - "futures-util", - "hmac 0.12.1", "http 1.3.1", "http-body 1.0.1", "hyper 1.6.0", "hyper-util", - "iam", - "include_dir", - "jsonwebtoken", "lazy_static", "libsystemd", - "lock", - "madmin", "matchit", - "mime", "mime_guess", - "netif", "opentelemetry", "percent-encoding", "pin-project-lite", - "policy", - "prost-build", - "protos", - "query", - "regex", "reqwest", - "rmp-serde", "rust-embed", + "rustfs-appauth", + "rustfs-common", "rustfs-config", + "rustfs-ecstore", "rustfs-filemeta", + "rustfs-iam", + "rustfs-madmin", "rustfs-notify", "rustfs-obs", + "rustfs-policy", + "rustfs-protos", "rustfs-rio", + "rustfs-s3select-api", + "rustfs-s3select-query", "rustfs-utils", "rustfs-zip", "rustls 0.23.28", @@ -8238,7 +7878,6 @@ dependencies = [ "serde", "serde_json", "serde_urlencoded", - "sha2 0.10.9", "shadow-rs", "socket2", "thiserror 2.0.12", @@ -8250,16 +7889,33 @@ dependencies = [ "tokio-tar", "tokio-util", "tonic", - "tonic-build", "tower", "tower-http", "tracing", - "transform-stream", "urlencoding", "uuid", "zip", ] +[[package]] +name = "rustfs-appauth" +version = "0.0.1" +dependencies = [ + "base64-simd", + "rsa", + "serde", + "serde_json", +] + +[[package]] +name = "rustfs-common" +version = "0.0.1" +dependencies = [ + "lazy_static", + "tokio", + "tonic", +] + [[package]] name = "rustfs-config" version = "0.0.1" @@ -8269,6 +7925,99 @@ dependencies = [ "serde_json", ] +[[package]] +name = "rustfs-crypto" +version = "0.0.1" +dependencies = [ + "aes-gcm", + "argon2", + "cfg-if", + "chacha20poly1305", + "jsonwebtoken", + "pbkdf2", + "rand 0.9.1", + "serde_json", + "sha2 0.10.9", + "test-case", + "thiserror 2.0.12", + "time", +] + +[[package]] +name = "rustfs-ecstore" +version = "0.0.1" +dependencies = [ + "async-channel", + "async-trait", + "aws-sdk-s3", + "base64 0.22.1", + "byteorder", + "bytes", + "bytesize", + "chrono", + "criterion", + "enumset", + "flatbuffers 25.2.10", + "futures", + "futures-util", + "glob", + "hex-simd", + "hmac 0.12.1", + "http 1.3.1", + "hyper 1.6.0", + "hyper-rustls 0.27.7", + "hyper-util", + "lazy_static", + "md-5", + "nix 0.30.1", + "num_cpus", + "once_cell", + "path-absolutize", + "path-clean", + "pin-project-lite", + "rand 0.9.1", + "reed-solomon-simd", + "regex", + "reqwest", + "rmp", + "rmp-serde", + "rustfs-common", + "rustfs-config", + "rustfs-filemeta", + "rustfs-lock", + "rustfs-madmin", + "rustfs-policy", + "rustfs-protos", + "rustfs-rio", + "rustfs-rsc", + "rustfs-signer", + "rustfs-utils", + "rustfs-workers", + "rustls 0.23.28", + "s3s", + "serde", + "serde-xml-rs 0.8.1", + "serde_json", + "sha2 0.10.9", + "shadow-rs", + "smallvec", + "temp-env", + "tempfile", + "thiserror 2.0.12", + "time", + "tokio", + "tokio-stream", + "tokio-util", + "tonic", + "tower", + "tracing", + "url", + "urlencoding", + "uuid", + "winapi", + "xxhash-rust", +] + [[package]] name = "rustfs-filemeta" version = "0.0.1" @@ -8311,6 +8060,59 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rustfs-iam" +version = "0.0.1" +dependencies = [ + "arc-swap", + "async-trait", + "base64-simd", + "futures", + "jsonwebtoken", + "lazy_static", + "rand 0.9.1", + "rustfs-crypto", + "rustfs-ecstore", + "rustfs-madmin", + "rustfs-policy", + "rustfs-utils", + "serde", + "serde_json", + "thiserror 2.0.12", + "time", + "tokio", + "tracing", +] + +[[package]] +name = "rustfs-lock" +version = "0.0.1" +dependencies = [ + "async-trait", + "lazy_static", + "rand 0.9.1", + "rustfs-protos", + "serde", + "serde_json", + "tokio", + "tonic", + "tracing", + "url", + "uuid", +] + +[[package]] +name = "rustfs-madmin" +version = "0.0.1" +dependencies = [ + "chrono", + "humantime", + "hyper 1.6.0", + "serde", + "serde_json", + "time", +] + [[package]] name = "rustfs-notify" version = "0.0.1" @@ -8319,13 +8121,13 @@ dependencies = [ "axum", "chrono", "dashmap 6.1.0", - "ecstore", "form_urlencoded", "once_cell", "quick-xml", "reqwest", "rumqttc", "rustfs-config", + "rustfs-ecstore", "rustfs-utils", "serde", "serde_json", @@ -8373,19 +8175,45 @@ dependencies = [ "tracing-subscriber", ] +[[package]] +name = "rustfs-policy" +version = "0.0.1" +dependencies = [ + "base64-simd", + "ipnetwork", + "jsonwebtoken", + "rand 0.9.1", + "regex", + "rustfs-crypto", + "serde", + "serde_json", + "strum", + "test-case", + "thiserror 2.0.12", + "time", + "tokio", +] + +[[package]] +name = "rustfs-protos" +version = "0.0.1" +dependencies = [ + "flatbuffers 25.2.10", + "prost", + "rustfs-common", + "tonic", + "tonic-build", +] + [[package]] name = "rustfs-rio" version = "0.0.1" dependencies = [ "aes-gcm", - "async-trait", - "base64-simd", - "byteorder", "bytes", "crc32fast", "criterion", "futures", - "hex-simd", "http 1.3.1", "md-5", "pin-project-lite", @@ -8426,6 +8254,48 @@ dependencies = [ "urlencoding", ] +[[package]] +name = "rustfs-s3select-api" +version = "0.0.1" +dependencies = [ + "async-trait", + "bytes", + "chrono", + "datafusion", + "futures", + "futures-core", + "http 1.3.1", + "object_store", + "pin-project-lite", + "rustfs-common", + "rustfs-ecstore", + "s3s", + "snafu", + "tokio", + "tokio-util", + "tracing", + "transform-stream", + "url", +] + +[[package]] +name = "rustfs-s3select-query" +version = "0.0.1" +dependencies = [ + "async-recursion", + "async-trait", + "datafusion", + "derive_builder", + "futures", + "lazy_static", + "parking_lot", + "rustfs-s3select-api", + "s3s", + "snafu", + "tokio", + "tracing", +] + [[package]] name = "rustfs-signer" version = "0.0.1" @@ -8434,11 +8304,8 @@ dependencies = [ "http 1.3.1", "hyper 1.6.0", "lazy_static", - "rand 0.9.1", "rustfs-utils", - "serde", "serde_urlencoded", - "tempfile", "time", "tracing", ] @@ -8451,7 +8318,6 @@ dependencies = [ "blake3", "brotli 8.0.1", "bytes", - "common", "crc32fast", "flate2", "futures", @@ -8488,17 +8354,22 @@ dependencies = [ "zstd", ] +[[package]] +name = "rustfs-workers" +version = "0.0.1" +dependencies = [ + "tokio", + "tracing", +] + [[package]] name = "rustfs-zip" version = "0.0.1" dependencies = [ "async-compression", - "async_zip", "tokio", "tokio-stream", "tokio-tar", - "xz2", - "zip", ] [[package]] @@ -11490,15 +11361,6 @@ dependencies = [ "bitflags 2.9.1", ] -[[package]] -name = "workers" -version = "0.0.1" -dependencies = [ - "common", - "tokio", - "tracing", -] - [[package]] name = "wrapcenum-derive" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index 55532bca..791b9c47 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,28 +14,28 @@ [workspace] members = [ - "appauth", # Application authentication and authorization + "rustfs", # Core file system implementation "cli/rustfs-gui", # Graphical user interface client - "common/common", # Shared utilities and data structures - "crates/filemeta", # File metadata management - "common/lock", # Distributed locking implementation - "common/protos", # Protocol buffer definitions - "common/workers", # Worker thread pools and task scheduling + "crates/appauth", # Application authentication and authorization + "crates/common", # Shared utilities and data structures "crates/config", # Configuration management + "crates/crypto", # Cryptography and security features + "crates/ecstore", # Erasure coding storage implementation + "crates/e2e_test", # End-to-end test suite + "crates/filemeta", # File metadata management + "crates/iam", # Identity and Access Management + "crates/lock", # Distributed locking implementation + "crates/madmin", # Management dashboard and admin API interface "crates/notify", # Notification system for events "crates/obs", # Observability utilities + "crates/protos", # Protocol buffer definitions "crates/rio", # Rust I/O utilities and abstractions - "crates/utils", # Utility functions and helpers - "crates/zip", # ZIP file handling and compression + "crates/s3select-api", # S3 Select API interface + "crates/s3select-query", # S3 Select query engine "crates/signer", # client signer - "crypto", # Cryptography and security features - "ecstore", # Erasure coding storage implementation - "e2e_test", # End-to-end test suite - "iam", # Identity and Access Management - "madmin", # Management dashboard and admin API interface - "rustfs", # Core file system implementation - "s3select/api", # S3 Select API interface - "s3select/query", # S3 Select query engine + "crates/utils", # Utility functions and helpers + "crates/workers", # Worker thread pools and task scheduling + "crates/zip", # ZIP file handling and compression ] resolver = "2" @@ -53,17 +53,17 @@ unsafe_code = "deny" all = "warn" [workspace.dependencies] -api = { path = "./s3select/api", version = "0.0.1" } -appauth = { path = "./appauth", version = "0.0.1" } -common = { path = "./common/common", version = "0.0.1" } -crypto = { path = "./crypto", version = "0.0.1" } -ecstore = { path = "./ecstore", version = "0.0.1" } -iam = { path = "./iam", version = "0.0.1" } -lock = { path = "./common/lock", version = "0.0.1" } -madmin = { path = "./madmin", version = "0.0.1" } -policy = { path = "./policy", version = "0.0.1" } -protos = { path = "./common/protos", version = "0.0.1" } -query = { path = "./s3select/query", version = "0.0.1" } +rustfs-s3select-api = { path = "crates/s3select-api", version = "0.0.1" } +rustfs-appauth = { path = "crates/appauth", version = "0.0.1" } +rustfs-common = { path = "crates/common", version = "0.0.1" } +rustfs-crypto = { path = "crates/crypto", version = "0.0.1" } +rustfs-ecstore = { path = "crates/ecstore", version = "0.0.1" } +rustfs-iam = { path = "crates/iam", version = "0.0.1" } +rustfs-lock = { path = "crates/lock", version = "0.0.1" } +rustfs-madmin = { path = "crates/madmin", version = "0.0.1" } +rustfs-policy = { path = "crates/policy", version = "0.0.1" } +rustfs-protos = { path = "crates/protos", version = "0.0.1" } +rustfs-s3select-query = { path = "crates/s3select-query", version = "0.0.1" } rustfs = { path = "./rustfs", version = "0.0.1" } rustfs-zip = { path = "./crates/zip", version = "0.0.1" } rustfs-config = { path = "./crates/config", version = "0.0.1" } @@ -73,7 +73,7 @@ rustfs-utils = { path = "crates/utils", version = "0.0.1" } rustfs-rio = { path = "crates/rio", version = "0.0.1" } rustfs-filemeta = { path = "crates/filemeta", version = "0.0.1" } rustfs-signer = { path = "crates/signer", version = "0.0.1" } -workers = { path = "./common/workers", version = "0.0.1" } +rustfs-workers = { path = "crates/workers", version = "0.0.1" } aes-gcm = { version = "0.10.3", features = ["std"] } arc-swap = "1.7.1" argon2 = { version = "0.5.3", features = ["std"] } @@ -81,15 +81,14 @@ atoi = "2.0.0" async-channel = "2.3.1" async-recursion = "1.1.1" async-trait = "0.1.88" +async-compression = { version = "0.4.0" } atomic_enum = "0.3.0" aws-sdk-s3 = "1.95.0" axum = "0.8.4" axum-extra = "0.10.1" axum-server = { version = "0.7.2", features = ["tls-rustls"] } -backon = "1.5.1" base64-simd = "0.8.0" base64 = "0.22.1" -blake2 = "0.10.6" brotli = "8.0.1" bytes = { version = "1.10.1", features = ["serde"] } bytesize = "2.0.1" @@ -100,6 +99,7 @@ chrono = { version = "0.4.41", features = ["serde"] } clap = { version = "4.5.40", features = ["derive", "env"] } const-str = { version = "0.6.2", features = ["std", "proc"] } crc32fast = "1.4.2" +criterion = { version = "0.5", features = ["html_reports"] } dashmap = "6.1.0" datafusion = "46.0.1" derive_builder = "0.20.2" @@ -128,9 +128,7 @@ hyper-rustls = "0.27.5" http = "1.3.1" http-body = "1.0.1" humantime = "2.2.0" -include_dir = "0.7.4" ipnetwork = { version = "0.21.1", features = ["serde"] } -itertools = "0.14.0" jsonwebtoken = "9.3.1" keyring = { version = "3.6.2", features = [ "apple-native", @@ -143,7 +141,6 @@ local-ip-address = "0.6.5" lz4 = "1.28.1" matchit = "0.8.4" md-5 = "0.10.6" -mime = "0.3.17" mime_guess = "2.0.5" netif = "0.1.6" nix = { version = "0.30.1", features = ["fs"] } @@ -175,7 +172,6 @@ percent-encoding = "2.3.1" pin-project-lite = "0.2.16" prost = "0.13.5" prost-build = "0.13.5" -protobuf = "3.7" quick-xml = "0.37.5" rand = "0.9.1" rdkafka = { version = "0.37.0", features = ["tokio"] } @@ -205,7 +201,6 @@ rustls = { version = "0.23.28" } rustls-pki-types = "1.12.0" rustls-pemfile = "2.2.0" s3s = { git = "https://github.com/Nugine/s3s.git", rev = "4733cdfb27b2713e832967232cbff413bb768c10" } -scopeguard = "1.2.0" shadow-rs = { version = "1.2.0", default-features = false } serde = { version = "1.0.219", features = ["derive"] } serde_json = { version = "1.0.140", features = ["raw_value"] } @@ -213,7 +208,6 @@ serde-xml-rs = "0.8.1" serde_urlencoded = "0.7.1" sha1 = "0.10.6" sha2 = "0.10.9" -std-next = "0.1.9" siphasher = "1.0.1" smallvec = { version = "1.15.1", features = ["serde"] } snafu = "0.8.6" diff --git a/common/common/src/error.rs b/common/common/src/error.rs deleted file mode 100644 index c06c380a..00000000 --- a/common/common/src/error.rs +++ /dev/null @@ -1,360 +0,0 @@ -// Copyright 2024 RustFS Team -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -use tracing_error::{SpanTrace, SpanTraceStatus}; - -pub type StdError = Box; - -pub type Result = std::result::Result; - -#[derive(Debug)] -pub struct Error { - inner: Box, - span_trace: SpanTrace, -} - -impl Error { - pub fn other(error: E) -> Self - where - E: std::fmt::Display + Into>, - { - Self::from_std_error(error.into()) - } - - /// Create a new error from a `std::error::Error`. - #[must_use] - #[track_caller] - pub fn new(source: T) -> Self { - Self::from_std_error(source.into()) - } - - /// Create a new error from a `std::error::Error`. - #[must_use] - #[track_caller] - pub fn from_std_error(inner: StdError) -> Self { - Self { - inner, - span_trace: SpanTrace::capture(), - } - } - - /// Create a new error from a string. - #[must_use] - #[track_caller] - pub fn from_string(s: impl Into) -> Self { - Self::msg(s) - } - - /// Create a new error from a string. - #[must_use] - #[track_caller] - pub fn msg(s: impl Into) -> Self { - Self::from_std_error(s.into().into()) - } - - /// Returns `true` if the inner type is the same as `T`. - #[inline] - pub fn is(&self) -> bool { - self.inner.is::() - } - - /// Returns some reference to the inner value if it is of type `T`, or - /// `None` if it isn't. - #[inline] - pub fn downcast_ref(&self) -> Option<&T> { - self.inner.downcast_ref() - } - - /// Returns some mutable reference to the inner value if it is of type `T`, or - /// `None` if it isn't. - #[inline] - pub fn downcast_mut(&mut self) -> Option<&mut T> { - self.inner.downcast_mut() - } - - pub fn to_io_err(&self) -> Option { - self.downcast_ref::() - .map(|e| std::io::Error::new(e.kind(), e.to_string())) - } - - pub fn inner_string(&self) -> String { - self.inner.to_string() - } -} - -impl From for Error { - fn from(e: T) -> Self { - Self::new(e) - } -} - -impl std::fmt::Display for Error { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}", self.inner)?; - - if self.span_trace.status() != SpanTraceStatus::EMPTY { - write!(f, "\nspan_trace:\n{}", self.span_trace)?; - } - - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io; - - #[derive(Debug)] - struct CustomTestError { - message: String, - } - - impl std::fmt::Display for CustomTestError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Custom test error: {}", self.message) - } - } - - impl std::error::Error for CustomTestError {} - - #[derive(Debug)] - struct AnotherTestError; - - impl std::fmt::Display for AnotherTestError { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "Another test error") - } - } - - impl std::error::Error for AnotherTestError {} - - #[test] - fn test_error_new_from_std_error() { - let io_error = io::Error::new(io::ErrorKind::NotFound, "File not found"); - let error = Error::new(io_error); - - assert!(error.inner_string().contains("File not found")); - assert!(error.is::()); - } - - #[test] - fn test_error_from_std_error() { - let io_error = io::Error::new(io::ErrorKind::PermissionDenied, "Permission denied"); - let boxed_error: StdError = Box::new(io_error); - let error = Error::from_std_error(boxed_error); - - assert!(error.inner_string().contains("Permission denied")); - assert!(error.is::()); - } - - #[test] - fn test_error_from_string() { - let error = Error::from_string("Test error message"); - assert_eq!(error.inner_string(), "Test error message"); - } - - #[test] - fn test_error_msg() { - let error = Error::msg("Another test message"); - assert_eq!(error.inner_string(), "Another test message"); - } - - #[test] - fn test_error_msg_with_string() { - let message = String::from("String message"); - let error = Error::msg(message); - assert_eq!(error.inner_string(), "String message"); - } - - #[test] - fn test_error_is_type_checking() { - let io_error = io::Error::new(io::ErrorKind::InvalidInput, "Invalid input"); - let error = Error::new(io_error); - - assert!(error.is::()); - assert!(!error.is::()); - } - - #[test] - fn test_error_downcast_ref() { - let io_error = io::Error::new(io::ErrorKind::TimedOut, "Operation timed out"); - let error = Error::new(io_error); - - let downcast_io = error.downcast_ref::(); - assert!(downcast_io.is_some()); - assert_eq!(downcast_io.unwrap().kind(), io::ErrorKind::TimedOut); - - let downcast_custom = error.downcast_ref::(); - assert!(downcast_custom.is_none()); - } - - #[test] - fn test_error_downcast_mut() { - let io_error = io::Error::new(io::ErrorKind::Interrupted, "Operation interrupted"); - let mut error = Error::new(io_error); - - let downcast_io = error.downcast_mut::(); - assert!(downcast_io.is_some()); - assert_eq!(downcast_io.unwrap().kind(), io::ErrorKind::Interrupted); - - let downcast_custom = error.downcast_mut::(); - assert!(downcast_custom.is_none()); - } - - #[test] - fn test_error_to_io_err() { - // Test with IO error - let original_io_error = io::Error::new(io::ErrorKind::BrokenPipe, "Broken pipe"); - let error = Error::new(original_io_error); - - let converted_io_error = error.to_io_err(); - assert!(converted_io_error.is_some()); - let io_err = converted_io_error.unwrap(); - assert_eq!(io_err.kind(), io::ErrorKind::BrokenPipe); - assert!(io_err.to_string().contains("Broken pipe")); - - // Test with non-IO error - let custom_error = CustomTestError { - message: "Not an IO error".to_string(), - }; - let error = Error::new(custom_error); - - let converted_io_error = error.to_io_err(); - assert!(converted_io_error.is_none()); - } - - #[test] - fn test_error_inner_string() { - let custom_error = CustomTestError { - message: "Test message".to_string(), - }; - let error = Error::new(custom_error); - - assert_eq!(error.inner_string(), "Custom test error: Test message"); - } - - #[test] - fn test_error_from_trait() { - let io_error = io::Error::new(io::ErrorKind::UnexpectedEof, "Unexpected EOF"); - let error: Error = io_error.into(); - - assert!(error.is::()); - assert!(error.inner_string().contains("Unexpected EOF")); - } - - #[test] - fn test_error_display() { - let custom_error = CustomTestError { - message: "Display test".to_string(), - }; - let error = Error::new(custom_error); - - let display_string = format!("{}", error); - assert!(display_string.contains("Custom test error: Display test")); - } - - #[test] - fn test_error_debug() { - let error = Error::msg("Debug test"); - let debug_string = format!("{:?}", error); - - assert!(debug_string.contains("Error")); - assert!(debug_string.contains("inner")); - assert!(debug_string.contains("span_trace")); - } - - #[test] - fn test_multiple_error_types() { - let errors = vec![ - Error::new(io::Error::new(io::ErrorKind::NotFound, "Not found")), - Error::new(CustomTestError { - message: "Custom".to_string(), - }), - Error::new(AnotherTestError), - Error::msg("String error"), - ]; - - assert!(errors[0].is::()); - assert!(errors[1].is::()); - assert!(errors[2].is::()); - assert!(!errors[3].is::()); - } - - #[test] - fn test_error_chain_compatibility() { - // Test that our Error type works well with error chains - let io_error = io::Error::new(io::ErrorKind::InvalidData, "Invalid data"); - let error = Error::new(io_error); - - // Should be able to convert back to Result - let result: Result<(), Error> = Err(error); - assert!(result.is_err()); - - // Test the error from the result - if let Err(err) = result { - assert!(err.is::()); - } - } - - #[test] - fn test_result_type_alias() { - // Test the Result type alias - fn test_function() -> Result { - Ok("Success".to_string()) - } - - fn test_function_with_error() -> Result { - Err(Error::msg("Test error")) - } - - let success_result = test_function(); - assert!(success_result.is_ok()); - assert_eq!(success_result.unwrap(), "Success"); - - let error_result = test_function_with_error(); - assert!(error_result.is_err()); - assert_eq!(error_result.unwrap_err().inner_string(), "Test error"); - } - - #[test] - fn test_error_with_empty_message() { - let error = Error::msg(""); - assert_eq!(error.inner_string(), ""); - } - - #[test] - fn test_error_with_unicode_message() { - let unicode_message = "错误信息 🚨 Error message with émojis and ñon-ASCII"; - let error = Error::msg(unicode_message); - assert_eq!(error.inner_string(), unicode_message); - } - - #[test] - fn test_error_with_very_long_message() { - let long_message = "A".repeat(10000); - let error = Error::msg(&long_message); - assert_eq!(error.inner_string(), long_message); - } - - #[test] - fn test_span_trace_capture() { - // Test that span trace is captured (though we can't easily test the content) - let error = Error::msg("Span trace test"); - let display_string = format!("{}", error); - - // The error should at least contain the message - assert!(display_string.contains("Span trace test")); - } -} diff --git a/appauth/Cargo.toml b/crates/appauth/Cargo.toml similarity index 90% rename from appauth/Cargo.toml rename to crates/appauth/Cargo.toml index c587b373..05f0d4c6 100644 --- a/appauth/Cargo.toml +++ b/crates/appauth/Cargo.toml @@ -13,7 +13,7 @@ # limitations under the License. [package] -name = "appauth" +name = "rustfs-appauth" edition.workspace = true license.workspace = true repository.workspace = true @@ -22,9 +22,6 @@ version.workspace = true [dependencies] base64-simd = { workspace = true } -common.workspace = true -hex-simd = { workspace = true } -rand.workspace = true rsa = { workspace = true } serde.workspace = true serde_json.workspace = true diff --git a/appauth/src/lib.rs b/crates/appauth/src/lib.rs similarity index 100% rename from appauth/src/lib.rs rename to crates/appauth/src/lib.rs diff --git a/appauth/src/token.rs b/crates/appauth/src/token.rs similarity index 100% rename from appauth/src/token.rs rename to crates/appauth/src/token.rs diff --git a/common/common/Cargo.toml b/crates/common/Cargo.toml similarity index 87% rename from common/common/Cargo.toml rename to crates/common/Cargo.toml index 3d490752..f6d9910c 100644 --- a/common/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -13,17 +13,17 @@ # limitations under the License. [package] -name = "common" +name = "rustfs-common" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true [lints] workspace = true [dependencies] -async-trait.workspace = true lazy_static.workspace = true -scopeguard = { workspace = true } tokio.workspace = true tonic = { workspace = true } -tracing-error.workspace = true diff --git a/common/common/src/bucket_stats.rs b/crates/common/src/bucket_stats.rs similarity index 93% rename from common/common/src/bucket_stats.rs rename to crates/common/src/bucket_stats.rs index 4685e1e8..d07aee1b 100644 --- a/common/common/src/bucket_stats.rs +++ b/crates/common/src/bucket_stats.rs @@ -16,18 +16,18 @@ use std::collections::HashMap; use crate::last_minute::{self}; pub struct ReplicationLatency { - // 单个和多部分 PUT 请求的延迟 + // Delays for single and multipart PUT requests upload_histogram: last_minute::LastMinuteHistogram, } impl ReplicationLatency { - // 合并两个 ReplicationLatency + // Merge two ReplicationLatency pub fn merge(&mut self, other: &mut ReplicationLatency) -> &ReplicationLatency { self.upload_histogram.merge(&other.upload_histogram); self } - // 获取上传延迟（按对象大小区间分类） + // Get upload delay (categorized by object size interval) pub fn get_upload_latency(&mut self) -> HashMap { let mut ret = HashMap::new(); let avg = self.upload_histogram.get_avg_data(); @@ -41,7 +41,7 @@ impl ReplicationLatency { self.upload_histogram.add(size, during); } - // 模拟从 size tag 到字符串的转换 + // Simulate the conversion from size tag to string fn size_tag_to_string(&self, tag: usize) -> String { match tag { 0 => String::from("Size < 1 KiB"), diff --git a/common/common/src/globals.rs b/crates/common/src/globals.rs similarity index 100% rename from common/common/src/globals.rs rename to crates/common/src/globals.rs diff --git a/common/common/src/last_minute.rs b/crates/common/src/last_minute.rs similarity index 100% rename from common/common/src/last_minute.rs rename to crates/common/src/last_minute.rs diff --git a/common/common/src/lib.rs b/crates/common/src/lib.rs similarity index 100% rename from common/common/src/lib.rs rename to crates/common/src/lib.rs diff --git a/crypto/Cargo.toml b/crates/crypto/Cargo.toml similarity index 98% rename from crypto/Cargo.toml rename to crates/crypto/Cargo.toml index 7d341ffe..8b0943f9 100644 --- a/crypto/Cargo.toml +++ b/crates/crypto/Cargo.toml @@ -13,7 +13,7 @@ # limitations under the License. [package] -name = "crypto" +name = "rustfs-crypto" edition.workspace = true license.workspace = true repository.workspace = true diff --git a/crypto/src/encdec.rs b/crates/crypto/src/encdec.rs similarity index 100% rename from crypto/src/encdec.rs rename to crates/crypto/src/encdec.rs diff --git a/crypto/src/encdec/aes.rs b/crates/crypto/src/encdec/aes.rs similarity index 100% rename from crypto/src/encdec/aes.rs rename to crates/crypto/src/encdec/aes.rs diff --git a/crypto/src/encdec/decrypt.rs b/crates/crypto/src/encdec/decrypt.rs similarity index 100% rename from crypto/src/encdec/decrypt.rs rename to crates/crypto/src/encdec/decrypt.rs diff --git a/crypto/src/encdec/encrypt.rs b/crates/crypto/src/encdec/encrypt.rs similarity index 100% rename from crypto/src/encdec/encrypt.rs rename to crates/crypto/src/encdec/encrypt.rs diff --git a/crypto/src/encdec/id.rs b/crates/crypto/src/encdec/id.rs similarity index 100% rename from crypto/src/encdec/id.rs rename to crates/crypto/src/encdec/id.rs diff --git a/crypto/src/encdec/tests.rs b/crates/crypto/src/encdec/tests.rs similarity index 100% rename from crypto/src/encdec/tests.rs rename to crates/crypto/src/encdec/tests.rs diff --git a/crypto/src/error.rs b/crates/crypto/src/error.rs similarity index 100% rename from crypto/src/error.rs rename to crates/crypto/src/error.rs diff --git a/crypto/src/jwt.rs b/crates/crypto/src/jwt.rs similarity index 100% rename from crypto/src/jwt.rs rename to crates/crypto/src/jwt.rs diff --git a/crypto/src/jwt/decode.rs b/crates/crypto/src/jwt/decode.rs similarity index 100% rename from crypto/src/jwt/decode.rs rename to crates/crypto/src/jwt/decode.rs diff --git a/crypto/src/jwt/encode.rs b/crates/crypto/src/jwt/encode.rs similarity index 100% rename from crypto/src/jwt/encode.rs rename to crates/crypto/src/jwt/encode.rs diff --git a/crypto/src/jwt/tests.rs b/crates/crypto/src/jwt/tests.rs similarity index 100% rename from crypto/src/jwt/tests.rs rename to crates/crypto/src/jwt/tests.rs diff --git a/crypto/src/lib.rs b/crates/crypto/src/lib.rs similarity index 100% rename from crypto/src/lib.rs rename to crates/crypto/src/lib.rs diff --git a/e2e_test/Cargo.toml b/crates/e2e_test/Cargo.toml similarity index 79% rename from e2e_test/Cargo.toml rename to crates/e2e_test/Cargo.toml index 44296391..b93ff488 100644 --- a/e2e_test/Cargo.toml +++ b/crates/e2e_test/Cargo.toml @@ -20,27 +20,21 @@ license.workspace = true repository.workspace = true rust-version.workspace = true - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [lints] workspace = true [dependencies] -ecstore.workspace = true +rustfs-ecstore.workspace = true flatbuffers.workspace = true futures.workspace = true -lazy_static.workspace = true -lock.workspace = true -protos.workspace = true +rustfs-lock.workspace = true +rustfs-protos.workspace = true rmp-serde.workspace = true serde.workspace = true serde_json.workspace = true tonic = { workspace = true } tokio = { workspace = true } -tower.workspace = true url.workspace = true -madmin.workspace =true -common.workspace = true +rustfs-madmin.workspace = true rustfs-filemeta.workspace = true bytes.workspace = true diff --git a/e2e_test/README.md b/crates/e2e_test/README.md similarity index 100% rename from e2e_test/README.md rename to crates/e2e_test/README.md diff --git a/e2e_test/src/lib.rs b/crates/e2e_test/src/lib.rs similarity index 100% rename from e2e_test/src/lib.rs rename to crates/e2e_test/src/lib.rs diff --git a/e2e_test/src/reliant/README.md b/crates/e2e_test/src/reliant/README.md similarity index 100% rename from e2e_test/src/reliant/README.md rename to crates/e2e_test/src/reliant/README.md diff --git a/e2e_test/src/reliant/lock.rs b/crates/e2e_test/src/reliant/lock.rs similarity index 95% rename from e2e_test/src/reliant/lock.rs rename to crates/e2e_test/src/reliant/lock.rs index 078d9aef..3b8b0441 100644 --- a/e2e_test/src/reliant/lock.rs +++ b/crates/e2e_test/src/reliant/lock.rs @@ -13,15 +13,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{error::Error, sync::Arc, time::Duration}; - -use lock::{ +use rustfs_lock::{ drwmutex::Options, lock_args::LockArgs, namespace_lock::{NsLockMap, new_nslock}, new_lock_api, }; -use protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest}; +use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest}; +use std::{error::Error, sync::Arc, time::Duration}; use tokio::sync::RwLock; use tonic::Request; diff --git a/e2e_test/src/reliant/mod.rs b/crates/e2e_test/src/reliant/mod.rs similarity index 100% rename from e2e_test/src/reliant/mod.rs rename to crates/e2e_test/src/reliant/mod.rs diff --git a/e2e_test/src/reliant/node_interact_test.rs b/crates/e2e_test/src/reliant/node_interact_test.rs similarity index 96% rename from e2e_test/src/reliant/node_interact_test.rs rename to crates/e2e_test/src/reliant/node_interact_test.rs index 88d34f70..7b176931 100644 --- a/e2e_test/src/reliant/node_interact_test.rs +++ b/crates/e2e_test/src/reliant/node_interact_test.rs @@ -13,21 +13,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::disk::{VolumeInfo, WalkDirOptions}; - use futures::future::join_all; -use protos::proto_gen::node_service::WalkDirRequest; -use protos::{ +use rmp_serde::{Deserializer, Serializer}; +use rustfs_ecstore::disk::{VolumeInfo, WalkDirOptions}; +use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter}; +use rustfs_protos::proto_gen::node_service::WalkDirRequest; +use rustfs_protos::{ models::{PingBody, PingBodyBuilder}, node_service_time_out_client, proto_gen::node_service::{ ListVolumesRequest, LocalStorageInfoRequest, MakeVolumeRequest, PingRequest, PingResponse, ReadAllRequest, }, }; -use rmp_serde::{Deserializer, Serializer}; -use rustfs_filemeta::{MetaCacheEntry, MetacacheReader, MetacacheWriter}; use serde::{Deserialize, Serialize}; -use std::{error::Error, io::Cursor}; +use std::error::Error; +use std::io::Cursor; use tokio::spawn; use tonic::Request; use tonic::codegen::tokio_stream::StreamExt; @@ -199,7 +199,7 @@ async fn storage_info() -> Result<(), Box> { let info = response.storage_info; let mut buf = Deserializer::new(Cursor::new(info)); - let storage_info: madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap(); + let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf).unwrap(); println!("{storage_info:?}"); Ok(()) } diff --git a/ecstore/Cargo.toml b/crates/ecstore/Cargo.toml similarity index 93% rename from ecstore/Cargo.toml rename to crates/ecstore/Cargo.toml index 29f40b2b..63e12bae 100644 --- a/ecstore/Cargo.toml +++ b/crates/ecstore/Cargo.toml @@ -13,7 +13,7 @@ # limitations under the License. [package] -name = "ecstore" +name = "rustfs-ecstore" version.workspace = true edition.workspace = true license.workspace = true @@ -32,8 +32,8 @@ rustfs-config = { workspace = true, features = ["constants", "notify"] } async-trait.workspace = true bytes.workspace = true byteorder = { workspace = true } -common.workspace = true -policy.workspace = true +rustfs-common.workspace = true +rustfs-policy.workspace = true chrono.workspace = true glob = { workspace = true } thiserror.workspace = true @@ -51,10 +51,10 @@ url.workspace = true uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] } reed-solomon-simd = { workspace = true } lazy_static.workspace = true -lock.workspace = true +rustfs-lock.workspace = true regex = { workspace = true } path-absolutize = { workspace = true } -protos.workspace = true +rustfs-protos.workspace = true rmp.workspace = true rmp-serde.workspace = true tokio-util = { workspace = true, features = ["io", "compat"] } @@ -79,8 +79,8 @@ num_cpus = { workspace = true } rand.workspace = true pin-project-lite.workspace = true md-5.workspace = true -madmin.workspace = true -workers.workspace = true +rustfs-madmin.workspace = true +rustfs-workers.workspace = true reqwest = { workspace = true } aws-sdk-s3 = { workspace = true } once_cell = { workspace = true } diff --git a/ecstore/README_cn.md b/crates/ecstore/README_cn.md similarity index 85% rename from ecstore/README_cn.md rename to crates/ecstore/README_cn.md index 27ca55b4..d3ff83df 100644 --- a/ecstore/README_cn.md +++ b/crates/ecstore/README_cn.md @@ -1,6 +1,7 @@ # ECStore - Erasure Coding Storage -ECStore provides erasure coding functionality for the RustFS project, using high-performance Reed-Solomon SIMD implementation for optimal performance. +ECStore provides erasure coding functionality for the RustFS project, using high-performance Reed-Solomon SIMD +implementation for optimal performance. ## Reed-Solomon Implementation @@ -14,7 +15,7 @@ ECStore provides erasure coding functionality for the RustFS project, using high ### Usage Example ```rust -use ecstore::erasure_coding::Erasure; +use rustfs_ecstore::erasure_coding::Erasure; // Create erasure coding instance // 4 data shards, 2 parity shards, 1KB block size @@ -22,22 +23,22 @@ let erasure = Erasure::new(4, 2, 1024); // Encode data let data = b"hello world from rustfs erasure coding"; -let shards = erasure.encode_data(data)?; +let shards = erasure.encode_data(data) ?; // Simulate loss of one shard -let mut shards_opt: Vec>> = shards - .iter() - .map(|b| Some(b.to_vec())) - .collect(); +let mut shards_opt: Vec > > = shards +.iter() +.map( | b| Some(b.to_vec())) +.collect(); shards_opt[2] = None; // Lose shard 2 // Reconstruct missing data -erasure.decode_data(&mut shards_opt)?; +erasure.decode_data( & mut shards_opt) ?; // Recover original data let mut recovered = Vec::new(); for shard in shards_opt.iter().take(4) { // Only data shards - recovered.extend_from_slice(shard.as_ref().unwrap()); +recovered.extend_from_slice(shard.as_ref().unwrap()); } recovered.truncate(data.len()); assert_eq!(&recovered, data); @@ -46,6 +47,7 @@ assert_eq!(&recovered, data); ## Performance Considerations ### SIMD Implementation Benefits + - **High Throughput**: Optimized for large block sizes (>= 1KB recommended) - **CPU Optimization**: Leverages modern CPU SIMD instructions - **Scalability**: Excellent performance for high-throughput scenarios @@ -53,6 +55,7 @@ assert_eq!(&recovered, data); ### Implementation Details #### `reed-solomon-simd` + - **Instance Caching**: Encoder/decoder instances are cached and reused for optimal performance - **Thread Safety**: Thread-safe with RwLock-based caching - **SIMD Optimization**: Leverages CPU SIMD instructions for maximum performance @@ -68,15 +71,18 @@ assert_eq!(&recovered, data); ## Cross-Platform Compatibility The SIMD implementation supports: + - x86_64 with advanced SIMD instructions (AVX2, SSE) - aarch64 (ARM64) with NEON SIMD optimizations - Other architectures with fallback implementations -The implementation automatically selects the best available SIMD instructions for the target platform, providing optimal performance across different architectures. +The implementation automatically selects the best available SIMD instructions for the target platform, providing optimal +performance across different architectures. ## Testing and Benchmarking Run performance benchmarks: + ```bash # Run erasure coding benchmarks cargo bench --bench erasure_benchmark @@ -91,6 +97,7 @@ cargo bench --bench comparison_benchmark ## Error Handling All operations return `Result` types with comprehensive error information: + - Encoding errors: Invalid parameters, insufficient memory - Decoding errors: Too many missing shards, corrupted data - Configuration errors: Invalid shard counts, unsupported parameters \ No newline at end of file diff --git a/ecstore/benches/comparison_benchmark.rs b/crates/ecstore/benches/comparison_benchmark.rs similarity index 99% rename from ecstore/benches/comparison_benchmark.rs rename to crates/ecstore/benches/comparison_benchmark.rs index 102a57c9..72e5b5f2 100644 --- a/ecstore/benches/comparison_benchmark.rs +++ b/crates/ecstore/benches/comparison_benchmark.rs @@ -33,7 +33,7 @@ //! ``` use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; -use ecstore::erasure_coding::Erasure; +use rustfs_ecstore::erasure_coding::Erasure; use std::time::Duration; /// Performance test data configuration diff --git a/ecstore/benches/erasure_benchmark.rs b/crates/ecstore/benches/erasure_benchmark.rs similarity index 99% rename from ecstore/benches/erasure_benchmark.rs rename to crates/ecstore/benches/erasure_benchmark.rs index 08ce4693..72e2f68d 100644 --- a/ecstore/benches/erasure_benchmark.rs +++ b/crates/ecstore/benches/erasure_benchmark.rs @@ -44,7 +44,7 @@ //! - SIMD optimization for different shard sizes use criterion::{BenchmarkId, Criterion, Throughput, black_box, criterion_group, criterion_main}; -use ecstore::erasure_coding::{Erasure, calc_shard_size}; +use rustfs_ecstore::erasure_coding::{Erasure, calc_shard_size}; use std::time::Duration; /// Benchmark configuration structure diff --git a/ecstore/build.rs b/crates/ecstore/build.rs similarity index 100% rename from ecstore/build.rs rename to crates/ecstore/build.rs diff --git a/ecstore/run_benchmarks.sh b/crates/ecstore/run_benchmarks.sh similarity index 100% rename from ecstore/run_benchmarks.sh rename to crates/ecstore/run_benchmarks.sh diff --git a/ecstore/src/admin_server_info.rs b/crates/ecstore/src/admin_server_info.rs similarity index 94% rename from ecstore/src/admin_server_info.rs rename to crates/ecstore/src/admin_server_info.rs index 01fc157f..977cb6fe 100644 --- a/ecstore/src/admin_server_info.rs +++ b/crates/ecstore/src/admin_server_info.rs @@ -25,12 +25,14 @@ use crate::{ notification_sys::get_global_notification_sys, store_api::StorageAPI, }; -use common::{ +use rustfs_common::{ // error::{Error, Result}, globals::GLOBAL_Local_Node_Name, }; -use madmin::{BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, ServerProperties}; -use protos::{ +use rustfs_madmin::{ + BackendDisks, Disk, ErasureSetInfo, ITEM_INITIALIZING, ITEM_OFFLINE, ITEM_ONLINE, InfoMessage, ServerProperties, +}; +use rustfs_protos::{ models::{PingBody, PingBodyBuilder}, node_service_time_out_client, proto_gen::node_service::{PingRequest, PingResponse}, @@ -219,13 +221,13 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage { warn!("server_info end {:?}", after2 - after1); servers.push(local); - let mut buckets = madmin::Buckets::default(); - let mut objects = madmin::Objects::default(); - let mut versions = madmin::Versions::default(); - let mut delete_markers = madmin::DeleteMarkers::default(); - let mut usage = madmin::Usage::default(); + let mut buckets = rustfs_madmin::Buckets::default(); + let mut objects = rustfs_madmin::Objects::default(); + let mut versions = rustfs_madmin::Versions::default(); + let mut delete_markers = rustfs_madmin::DeleteMarkers::default(); + let mut usage = rustfs_madmin::Usage::default(); let mut mode = ITEM_INITIALIZING; - let mut backend = madmin::ErasureBackend::default(); + let mut backend = rustfs_madmin::ErasureBackend::default(); let mut pools: HashMap> = HashMap::new(); if let Some(store) = new_object_layer_fn() { @@ -266,8 +268,8 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage { let after5 = OffsetDateTime::now_utc(); warn!("get_online_offline_disks_stats end {:?}", after5 - after4); - backend = madmin::ErasureBackend { - backend_type: madmin::BackendType::ErasureType, + backend = rustfs_madmin::ErasureBackend { + backend_type: rustfs_madmin::BackendType::ErasureType, online_disks: online_disks.sum(), offline_disks: offline_disks.sum(), standard_sc_parity: backen_info.standard_sc_parity, @@ -283,7 +285,7 @@ pub async fn get_server_info(get_pools: bool) -> InfoMessage { } } - let services = madmin::Services::default(); + let services = rustfs_madmin::Services::default(); InfoMessage { mode: Some(mode.to_string()), diff --git a/ecstore/src/bitrot.rs b/crates/ecstore/src/bitrot.rs similarity index 100% rename from ecstore/src/bitrot.rs rename to crates/ecstore/src/bitrot.rs diff --git a/ecstore/src/bucket/error.rs b/crates/ecstore/src/bucket/error.rs similarity index 100% rename from ecstore/src/bucket/error.rs rename to crates/ecstore/src/bucket/error.rs diff --git a/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs similarity index 100% rename from ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs rename to crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_audit.rs diff --git a/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs b/crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs similarity index 100% rename from ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs rename to crates/ecstore/src/bucket/lifecycle/bucket_lifecycle_ops.rs diff --git a/ecstore/src/bucket/lifecycle/lifecycle.rs b/crates/ecstore/src/bucket/lifecycle/lifecycle.rs similarity index 100% rename from ecstore/src/bucket/lifecycle/lifecycle.rs rename to crates/ecstore/src/bucket/lifecycle/lifecycle.rs diff --git a/ecstore/src/bucket/lifecycle/mod.rs b/crates/ecstore/src/bucket/lifecycle/mod.rs similarity index 100% rename from ecstore/src/bucket/lifecycle/mod.rs rename to crates/ecstore/src/bucket/lifecycle/mod.rs diff --git a/ecstore/src/bucket/lifecycle/rule.rs b/crates/ecstore/src/bucket/lifecycle/rule.rs similarity index 100% rename from ecstore/src/bucket/lifecycle/rule.rs rename to crates/ecstore/src/bucket/lifecycle/rule.rs diff --git a/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs b/crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs similarity index 100% rename from ecstore/src/bucket/lifecycle/tier_last_day_stats.rs rename to crates/ecstore/src/bucket/lifecycle/tier_last_day_stats.rs diff --git a/ecstore/src/bucket/lifecycle/tier_sweeper.rs b/crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs similarity index 100% rename from ecstore/src/bucket/lifecycle/tier_sweeper.rs rename to crates/ecstore/src/bucket/lifecycle/tier_sweeper.rs diff --git a/ecstore/src/bucket/metadata.rs b/crates/ecstore/src/bucket/metadata.rs similarity index 99% rename from ecstore/src/bucket/metadata.rs rename to crates/ecstore/src/bucket/metadata.rs index 645b3ea8..63b158f5 100644 --- a/ecstore/src/bucket/metadata.rs +++ b/crates/ecstore/src/bucket/metadata.rs @@ -17,8 +17,8 @@ use super::{quota::BucketQuota, target::BucketTargets}; use super::object_lock::ObjectLockApi; use super::versioning::VersioningApi; use byteorder::{BigEndian, ByteOrder, LittleEndian}; -use policy::policy::BucketPolicy; use rmp_serde::Serializer as rmpSerializer; +use rustfs_policy::policy::BucketPolicy; use s3s::dto::{ BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, diff --git a/ecstore/src/bucket/metadata_sys.rs b/crates/ecstore/src/bucket/metadata_sys.rs similarity index 99% rename from ecstore/src/bucket/metadata_sys.rs rename to crates/ecstore/src/bucket/metadata_sys.rs index 5343511e..791134da 100644 --- a/ecstore/src/bucket/metadata_sys.rs +++ b/crates/ecstore/src/bucket/metadata_sys.rs @@ -21,7 +21,7 @@ use crate::global::{GLOBAL_Endpoints, is_dist_erasure, is_erasure, new_object_la use crate::heal::heal_commands::HealOpts; use crate::store::ECStore; use futures::future::join_all; -use policy::policy::BucketPolicy; +use rustfs_policy::policy::BucketPolicy; use s3s::dto::{ BucketLifecycleConfiguration, NotificationConfiguration, ObjectLockConfiguration, ReplicationConfiguration, ServerSideEncryptionConfiguration, Tagging, VersioningConfiguration, diff --git a/ecstore/src/bucket/mod.rs b/crates/ecstore/src/bucket/mod.rs similarity index 100% rename from ecstore/src/bucket/mod.rs rename to crates/ecstore/src/bucket/mod.rs diff --git a/ecstore/src/bucket/msgpkg.md b/crates/ecstore/src/bucket/msgpkg.md similarity index 100% rename from ecstore/src/bucket/msgpkg.md rename to crates/ecstore/src/bucket/msgpkg.md diff --git a/ecstore/src/bucket/object_lock/mod.rs b/crates/ecstore/src/bucket/object_lock/mod.rs similarity index 100% rename from ecstore/src/bucket/object_lock/mod.rs rename to crates/ecstore/src/bucket/object_lock/mod.rs diff --git a/ecstore/src/bucket/object_lock/objectlock.rs b/crates/ecstore/src/bucket/object_lock/objectlock.rs similarity index 100% rename from ecstore/src/bucket/object_lock/objectlock.rs rename to crates/ecstore/src/bucket/object_lock/objectlock.rs diff --git a/ecstore/src/bucket/object_lock/objectlock_sys.rs b/crates/ecstore/src/bucket/object_lock/objectlock_sys.rs similarity index 100% rename from ecstore/src/bucket/object_lock/objectlock_sys.rs rename to crates/ecstore/src/bucket/object_lock/objectlock_sys.rs diff --git a/ecstore/src/bucket/policy_sys.rs b/crates/ecstore/src/bucket/policy_sys.rs similarity index 96% rename from ecstore/src/bucket/policy_sys.rs rename to crates/ecstore/src/bucket/policy_sys.rs index 68995135..f06ae5ab 100644 --- a/ecstore/src/bucket/policy_sys.rs +++ b/crates/ecstore/src/bucket/policy_sys.rs @@ -14,7 +14,7 @@ use super::{error::BucketMetadataError, metadata_sys::get_bucket_metadata_sys}; use crate::error::Result; -use policy::policy::{BucketPolicy, BucketPolicyArgs}; +use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs}; use tracing::warn; pub struct PolicySys {} diff --git a/ecstore/src/bucket/quota/mod.rs b/crates/ecstore/src/bucket/quota/mod.rs similarity index 100% rename from ecstore/src/bucket/quota/mod.rs rename to crates/ecstore/src/bucket/quota/mod.rs diff --git a/ecstore/src/bucket/replication/datatypes.rs b/crates/ecstore/src/bucket/replication/datatypes.rs similarity index 100% rename from ecstore/src/bucket/replication/datatypes.rs rename to crates/ecstore/src/bucket/replication/datatypes.rs diff --git a/ecstore/src/bucket/replication/mod.rs b/crates/ecstore/src/bucket/replication/mod.rs similarity index 100% rename from ecstore/src/bucket/replication/mod.rs rename to crates/ecstore/src/bucket/replication/mod.rs diff --git a/ecstore/src/bucket/tagging/mod.rs b/crates/ecstore/src/bucket/tagging/mod.rs similarity index 100% rename from ecstore/src/bucket/tagging/mod.rs rename to crates/ecstore/src/bucket/tagging/mod.rs diff --git a/ecstore/src/bucket/target/mod.rs b/crates/ecstore/src/bucket/target/mod.rs similarity index 100% rename from ecstore/src/bucket/target/mod.rs rename to crates/ecstore/src/bucket/target/mod.rs diff --git a/ecstore/src/bucket/utils.rs b/crates/ecstore/src/bucket/utils.rs similarity index 100% rename from ecstore/src/bucket/utils.rs rename to crates/ecstore/src/bucket/utils.rs diff --git a/ecstore/src/bucket/versioning/mod.rs b/crates/ecstore/src/bucket/versioning/mod.rs similarity index 100% rename from ecstore/src/bucket/versioning/mod.rs rename to crates/ecstore/src/bucket/versioning/mod.rs diff --git a/ecstore/src/bucket/versioning_sys.rs b/crates/ecstore/src/bucket/versioning_sys.rs similarity index 100% rename from ecstore/src/bucket/versioning_sys.rs rename to crates/ecstore/src/bucket/versioning_sys.rs diff --git a/ecstore/src/cache_value/cache.rs b/crates/ecstore/src/cache_value/cache.rs similarity index 100% rename from ecstore/src/cache_value/cache.rs rename to crates/ecstore/src/cache_value/cache.rs diff --git a/ecstore/src/cache_value/metacache_set.rs b/crates/ecstore/src/cache_value/metacache_set.rs similarity index 100% rename from ecstore/src/cache_value/metacache_set.rs rename to crates/ecstore/src/cache_value/metacache_set.rs diff --git a/ecstore/src/cache_value/mod.rs b/crates/ecstore/src/cache_value/mod.rs similarity index 100% rename from ecstore/src/cache_value/mod.rs rename to crates/ecstore/src/cache_value/mod.rs diff --git a/ecstore/src/checksum.rs b/crates/ecstore/src/checksum.rs similarity index 100% rename from ecstore/src/checksum.rs rename to crates/ecstore/src/checksum.rs diff --git a/ecstore/src/chunk_stream.rs b/crates/ecstore/src/chunk_stream.rs similarity index 100% rename from ecstore/src/chunk_stream.rs rename to crates/ecstore/src/chunk_stream.rs diff --git a/ecstore/src/client/admin_handler_utils.rs b/crates/ecstore/src/client/admin_handler_utils.rs similarity index 100% rename from ecstore/src/client/admin_handler_utils.rs rename to crates/ecstore/src/client/admin_handler_utils.rs diff --git a/ecstore/src/client/api_bucket_policy.rs b/crates/ecstore/src/client/api_bucket_policy.rs similarity index 100% rename from ecstore/src/client/api_bucket_policy.rs rename to crates/ecstore/src/client/api_bucket_policy.rs diff --git a/ecstore/src/client/api_error_response.rs b/crates/ecstore/src/client/api_error_response.rs similarity index 100% rename from ecstore/src/client/api_error_response.rs rename to crates/ecstore/src/client/api_error_response.rs diff --git a/ecstore/src/client/api_get_object.rs b/crates/ecstore/src/client/api_get_object.rs similarity index 100% rename from ecstore/src/client/api_get_object.rs rename to crates/ecstore/src/client/api_get_object.rs diff --git a/ecstore/src/client/api_get_options.rs b/crates/ecstore/src/client/api_get_options.rs similarity index 100% rename from ecstore/src/client/api_get_options.rs rename to crates/ecstore/src/client/api_get_options.rs diff --git a/ecstore/src/client/api_list.rs b/crates/ecstore/src/client/api_list.rs similarity index 100% rename from ecstore/src/client/api_list.rs rename to crates/ecstore/src/client/api_list.rs diff --git a/ecstore/src/client/api_put_object.rs b/crates/ecstore/src/client/api_put_object.rs similarity index 100% rename from ecstore/src/client/api_put_object.rs rename to crates/ecstore/src/client/api_put_object.rs diff --git a/ecstore/src/client/api_put_object_common.rs b/crates/ecstore/src/client/api_put_object_common.rs similarity index 100% rename from ecstore/src/client/api_put_object_common.rs rename to crates/ecstore/src/client/api_put_object_common.rs diff --git a/ecstore/src/client/api_put_object_multipart.rs b/crates/ecstore/src/client/api_put_object_multipart.rs similarity index 100% rename from ecstore/src/client/api_put_object_multipart.rs rename to crates/ecstore/src/client/api_put_object_multipart.rs diff --git a/ecstore/src/client/api_put_object_streaming.rs b/crates/ecstore/src/client/api_put_object_streaming.rs similarity index 100% rename from ecstore/src/client/api_put_object_streaming.rs rename to crates/ecstore/src/client/api_put_object_streaming.rs diff --git a/ecstore/src/client/api_remove.rs b/crates/ecstore/src/client/api_remove.rs similarity index 100% rename from ecstore/src/client/api_remove.rs rename to crates/ecstore/src/client/api_remove.rs diff --git a/ecstore/src/client/api_s3_datatypes.rs b/crates/ecstore/src/client/api_s3_datatypes.rs similarity index 100% rename from ecstore/src/client/api_s3_datatypes.rs rename to crates/ecstore/src/client/api_s3_datatypes.rs diff --git a/ecstore/src/client/bucket_cache.rs b/crates/ecstore/src/client/bucket_cache.rs similarity index 100% rename from ecstore/src/client/bucket_cache.rs rename to crates/ecstore/src/client/bucket_cache.rs diff --git a/ecstore/src/client/constants.rs b/crates/ecstore/src/client/constants.rs similarity index 100% rename from ecstore/src/client/constants.rs rename to crates/ecstore/src/client/constants.rs diff --git a/ecstore/src/client/credentials.rs b/crates/ecstore/src/client/credentials.rs similarity index 100% rename from ecstore/src/client/credentials.rs rename to crates/ecstore/src/client/credentials.rs diff --git a/ecstore/src/client/hook_reader.rs b/crates/ecstore/src/client/hook_reader.rs similarity index 100% rename from ecstore/src/client/hook_reader.rs rename to crates/ecstore/src/client/hook_reader.rs diff --git a/ecstore/src/client/mod.rs b/crates/ecstore/src/client/mod.rs similarity index 100% rename from ecstore/src/client/mod.rs rename to crates/ecstore/src/client/mod.rs diff --git a/ecstore/src/client/object_api_utils.rs b/crates/ecstore/src/client/object_api_utils.rs similarity index 100% rename from ecstore/src/client/object_api_utils.rs rename to crates/ecstore/src/client/object_api_utils.rs diff --git a/ecstore/src/client/object_handlers_common.rs b/crates/ecstore/src/client/object_handlers_common.rs similarity index 97% rename from ecstore/src/client/object_handlers_common.rs rename to crates/ecstore/src/client/object_handlers_common.rs index 6aa98899..6c380aab 100644 --- a/ecstore/src/client/object_handlers_common.rs +++ b/crates/ecstore/src/client/object_handlers_common.rs @@ -18,7 +18,7 @@ use crate::bucket::versioning::VersioningApi; use crate::bucket::versioning_sys::BucketVersioningSys; use crate::store::ECStore; use crate::store_api::{ObjectOptions, ObjectToDelete}; -use lock::local_locker::MAX_DELETE_LIST; +use rustfs_lock::local_locker::MAX_DELETE_LIST; pub async fn delete_object_versions(api: ECStore, bucket: &str, to_del: &[ObjectToDelete], _lc_event: lifecycle::Event) { let mut remaining = to_del; diff --git a/ecstore/src/client/transition_api.rs b/crates/ecstore/src/client/transition_api.rs similarity index 100% rename from ecstore/src/client/transition_api.rs rename to crates/ecstore/src/client/transition_api.rs diff --git a/ecstore/src/client/utils.rs b/crates/ecstore/src/client/utils.rs similarity index 100% rename from ecstore/src/client/utils.rs rename to crates/ecstore/src/client/utils.rs diff --git a/ecstore/src/cmd/bucket_replication.rs b/crates/ecstore/src/cmd/bucket_replication.rs similarity index 100% rename from ecstore/src/cmd/bucket_replication.rs rename to crates/ecstore/src/cmd/bucket_replication.rs diff --git a/ecstore/src/cmd/bucket_replication_utils.rs b/crates/ecstore/src/cmd/bucket_replication_utils.rs similarity index 100% rename from ecstore/src/cmd/bucket_replication_utils.rs rename to crates/ecstore/src/cmd/bucket_replication_utils.rs diff --git a/ecstore/src/cmd/bucket_targets.rs b/crates/ecstore/src/cmd/bucket_targets.rs similarity index 100% rename from ecstore/src/cmd/bucket_targets.rs rename to crates/ecstore/src/cmd/bucket_targets.rs diff --git a/ecstore/src/cmd/bucketreplicationhandler.rs b/crates/ecstore/src/cmd/bucketreplicationhandler.rs similarity index 100% rename from ecstore/src/cmd/bucketreplicationhandler.rs rename to crates/ecstore/src/cmd/bucketreplicationhandler.rs diff --git a/ecstore/src/cmd/mod.rs b/crates/ecstore/src/cmd/mod.rs similarity index 100% rename from ecstore/src/cmd/mod.rs rename to crates/ecstore/src/cmd/mod.rs diff --git a/ecstore/src/compress.rs b/crates/ecstore/src/compress.rs similarity index 100% rename from ecstore/src/compress.rs rename to crates/ecstore/src/compress.rs diff --git a/ecstore/src/config/com.rs b/crates/ecstore/src/config/com.rs similarity index 100% rename from ecstore/src/config/com.rs rename to crates/ecstore/src/config/com.rs diff --git a/ecstore/src/config/heal.rs b/crates/ecstore/src/config/heal.rs similarity index 100% rename from ecstore/src/config/heal.rs rename to crates/ecstore/src/config/heal.rs diff --git a/ecstore/src/config/mod.rs b/crates/ecstore/src/config/mod.rs similarity index 100% rename from ecstore/src/config/mod.rs rename to crates/ecstore/src/config/mod.rs diff --git a/ecstore/src/config/notify.rs b/crates/ecstore/src/config/notify.rs similarity index 100% rename from ecstore/src/config/notify.rs rename to crates/ecstore/src/config/notify.rs diff --git a/ecstore/src/config/storageclass.rs b/crates/ecstore/src/config/storageclass.rs similarity index 100% rename from ecstore/src/config/storageclass.rs rename to crates/ecstore/src/config/storageclass.rs diff --git a/ecstore/src/disk/endpoint.rs b/crates/ecstore/src/disk/endpoint.rs similarity index 100% rename from ecstore/src/disk/endpoint.rs rename to crates/ecstore/src/disk/endpoint.rs diff --git a/ecstore/src/disk/error.rs b/crates/ecstore/src/disk/error.rs similarity index 99% rename from ecstore/src/disk/error.rs rename to crates/ecstore/src/disk/error.rs index 9d53ef9b..73718379 100644 --- a/ecstore/src/disk/error.rs +++ b/crates/ecstore/src/disk/error.rs @@ -253,8 +253,8 @@ impl From for DiskError { } } -impl From for DiskError { - fn from(e: protos::proto_gen::node_service::Error) -> Self { +impl From for DiskError { + fn from(e: rustfs_protos::proto_gen::node_service::Error) -> Self { if let Some(err) = DiskError::from_u32(e.code) { if matches!(err, DiskError::Io(_)) { DiskError::other(e.error_info) @@ -267,9 +267,9 @@ impl From for DiskError { } } -impl From for protos::proto_gen::node_service::Error { +impl From for rustfs_protos::proto_gen::node_service::Error { fn from(e: DiskError) -> Self { - protos::proto_gen::node_service::Error { + rustfs_protos::proto_gen::node_service::Error { code: e.to_u32(), error_info: e.to_string(), } diff --git a/ecstore/src/disk/error_conv.rs b/crates/ecstore/src/disk/error_conv.rs similarity index 100% rename from ecstore/src/disk/error_conv.rs rename to crates/ecstore/src/disk/error_conv.rs diff --git a/ecstore/src/disk/error_reduce.rs b/crates/ecstore/src/disk/error_reduce.rs similarity index 100% rename from ecstore/src/disk/error_reduce.rs rename to crates/ecstore/src/disk/error_reduce.rs diff --git a/ecstore/src/disk/format.rs b/crates/ecstore/src/disk/format.rs similarity index 100% rename from ecstore/src/disk/format.rs rename to crates/ecstore/src/disk/format.rs diff --git a/ecstore/src/disk/fs.rs b/crates/ecstore/src/disk/fs.rs similarity index 100% rename from ecstore/src/disk/fs.rs rename to crates/ecstore/src/disk/fs.rs diff --git a/ecstore/src/disk/local.rs b/crates/ecstore/src/disk/local.rs similarity index 99% rename from ecstore/src/disk/local.rs rename to crates/ecstore/src/disk/local.rs index 51dd9310..acb3b97a 100644 --- a/ecstore/src/disk/local.rs +++ b/crates/ecstore/src/disk/local.rs @@ -54,8 +54,8 @@ use tokio::time::interval; use crate::erasure_coding::bitrot_verify; use bytes::Bytes; -use common::defer; use path_absolutize::Absolutize; +use rustfs_common::defer; use rustfs_filemeta::{ Cache, FileInfo, FileInfoOpts, FileMeta, MetaCacheEntry, MetacacheWriter, Opts, RawFileInfo, UpdateFn, get_file_info, read_xl_meta_no_data, @@ -2663,7 +2663,7 @@ mod test { let valid_format_info = FormatInfo { id: Some(Uuid::new_v4()), data: vec![1, 2, 3].into(), - file_info: Some(fs::metadata(".").await.unwrap()), + file_info: Some(fs::metadata("../../../..").await.unwrap()), last_check: Some(now), }; assert!(valid_format_info.last_check_valid()); @@ -2672,7 +2672,7 @@ mod test { let invalid_format_info = FormatInfo { id: None, data: vec![1, 2, 3].into(), - file_info: Some(fs::metadata(".").await.unwrap()), + file_info: Some(fs::metadata("../../../..").await.unwrap()), last_check: Some(now), }; assert!(!invalid_format_info.last_check_valid()); @@ -2682,7 +2682,7 @@ mod test { let old_format_info = FormatInfo { id: Some(Uuid::new_v4()), data: vec![1, 2, 3].into(), - file_info: Some(fs::metadata(".").await.unwrap()), + file_info: Some(fs::metadata("../../../..").await.unwrap()), last_check: Some(old_time), }; assert!(!old_format_info.last_check_valid()); diff --git a/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs similarity index 99% rename from ecstore/src/disk/mod.rs rename to crates/ecstore/src/disk/mod.rs index 541100b2..617389ec 100644 --- a/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -41,8 +41,8 @@ use endpoint::Endpoint; use error::DiskError; use error::{Error, Result}; use local::LocalDisk; -use madmin::info_commands::DiskMetrics; use rustfs_filemeta::{FileInfo, RawFileInfo}; +use rustfs_madmin::info_commands::DiskMetrics; use serde::{Deserialize, Serialize}; use std::{fmt::Debug, path::PathBuf, sync::Arc}; use time::OffsetDateTime; diff --git a/ecstore/src/disk/os.rs b/crates/ecstore/src/disk/os.rs similarity index 100% rename from ecstore/src/disk/os.rs rename to crates/ecstore/src/disk/os.rs diff --git a/ecstore/src/disks_layout.rs b/crates/ecstore/src/disks_layout.rs similarity index 100% rename from ecstore/src/disks_layout.rs rename to crates/ecstore/src/disks_layout.rs diff --git a/ecstore/src/endpoints.rs b/crates/ecstore/src/endpoints.rs similarity index 100% rename from ecstore/src/endpoints.rs rename to crates/ecstore/src/endpoints.rs diff --git a/ecstore/src/erasure.rs b/crates/ecstore/src/erasure.rs similarity index 98% rename from ecstore/src/erasure.rs rename to crates/ecstore/src/erasure.rs index e8b13749..0b3c6cc2 100644 --- a/ecstore/src/erasure.rs +++ b/crates/ecstore/src/erasure.rs @@ -14,7 +14,7 @@ use crate::bitrot::{BitrotReader, BitrotWriter}; use crate::disk::error::{Error, Result}; -use crate::disk::error_reduce::{OBJECT_OP_IGNORED_ERRS, reduce_write_quorum_errs}; +use crate::disk::error_reduce::{reduce_write_quorum_errs, OBJECT_OP_IGNORED_ERRS}; use crate::io::Etag; use bytes::{Bytes, BytesMut}; use futures::future::join_all; @@ -22,17 +22,14 @@ use reed_solomon_erasure::galois_8::ReedSolomon; use smallvec::SmallVec; use std::any::Any; use std::io::ErrorKind; -use std::sync::Arc; +use std::sync::{mpsc, Arc}; use tokio::io::{AsyncRead, AsyncWrite}; use tokio::io::{AsyncReadExt, AsyncWriteExt}; use tokio::sync::mpsc; use tracing::warn; use tracing::{error, info}; -// use tracing::debug; use uuid::Uuid; -// use reader::reader::ChunkedStream; -// use crate::chunk_stream::ChunkedStream; use crate::disk::error::DiskError; #[derive(Default)] diff --git a/ecstore/src/erasure_coding/bitrot.rs b/crates/ecstore/src/erasure_coding/bitrot.rs similarity index 100% rename from ecstore/src/erasure_coding/bitrot.rs rename to crates/ecstore/src/erasure_coding/bitrot.rs diff --git a/ecstore/src/erasure_coding/decode.rs b/crates/ecstore/src/erasure_coding/decode.rs similarity index 100% rename from ecstore/src/erasure_coding/decode.rs rename to crates/ecstore/src/erasure_coding/decode.rs diff --git a/ecstore/src/erasure_coding/encode.rs b/crates/ecstore/src/erasure_coding/encode.rs similarity index 100% rename from ecstore/src/erasure_coding/encode.rs rename to crates/ecstore/src/erasure_coding/encode.rs diff --git a/ecstore/src/erasure_coding/erasure.rs b/crates/ecstore/src/erasure_coding/erasure.rs similarity index 100% rename from ecstore/src/erasure_coding/erasure.rs rename to crates/ecstore/src/erasure_coding/erasure.rs diff --git a/ecstore/src/erasure_coding/heal.rs b/crates/ecstore/src/erasure_coding/heal.rs similarity index 100% rename from ecstore/src/erasure_coding/heal.rs rename to crates/ecstore/src/erasure_coding/heal.rs diff --git a/ecstore/src/erasure_coding/mod.rs b/crates/ecstore/src/erasure_coding/mod.rs similarity index 100% rename from ecstore/src/erasure_coding/mod.rs rename to crates/ecstore/src/erasure_coding/mod.rs diff --git a/ecstore/src/error.rs b/crates/ecstore/src/error.rs similarity index 100% rename from ecstore/src/error.rs rename to crates/ecstore/src/error.rs diff --git a/ecstore/src/event/mod.rs b/crates/ecstore/src/event/mod.rs similarity index 100% rename from ecstore/src/event/mod.rs rename to crates/ecstore/src/event/mod.rs diff --git a/ecstore/src/event/name.rs b/crates/ecstore/src/event/name.rs similarity index 100% rename from ecstore/src/event/name.rs rename to crates/ecstore/src/event/name.rs diff --git a/ecstore/src/event/targetid.rs b/crates/ecstore/src/event/targetid.rs similarity index 100% rename from ecstore/src/event/targetid.rs rename to crates/ecstore/src/event/targetid.rs diff --git a/ecstore/src/event/targetlist.rs b/crates/ecstore/src/event/targetlist.rs similarity index 100% rename from ecstore/src/event/targetlist.rs rename to crates/ecstore/src/event/targetlist.rs diff --git a/ecstore/src/event_notification.rs b/crates/ecstore/src/event_notification.rs similarity index 100% rename from ecstore/src/event_notification.rs rename to crates/ecstore/src/event_notification.rs diff --git a/ecstore/src/global.rs b/crates/ecstore/src/global.rs similarity index 99% rename from ecstore/src/global.rs rename to crates/ecstore/src/global.rs index 329ed5eb..e809d2e9 100644 --- a/ecstore/src/global.rs +++ b/crates/ecstore/src/global.rs @@ -23,7 +23,7 @@ use crate::{ tier::tier::TierConfigMgr, }; use lazy_static::lazy_static; -use policy::auth::Credentials; +use rustfs_policy::auth::Credentials; use std::{ collections::HashMap, sync::{Arc, OnceLock}, diff --git a/ecstore/src/heal/background_heal_ops.rs b/crates/ecstore/src/heal/background_heal_ops.rs similarity index 99% rename from ecstore/src/heal/background_heal_ops.rs rename to crates/ecstore/src/heal/background_heal_ops.rs index 75727263..e447a585 100644 --- a/ecstore/src/heal/background_heal_ops.rs +++ b/crates/ecstore/src/heal/background_heal_ops.rs @@ -13,7 +13,7 @@ // limitations under the License. use futures::future::join_all; -use madmin::heal_commands::HealResultItem; +use rustfs_madmin::heal_commands::HealResultItem; use rustfs_utils::path::{SLASH_SEPARATOR, path_join}; use std::{cmp::Ordering, env, path::PathBuf, sync::Arc, time::Duration}; use tokio::{ diff --git a/ecstore/src/heal/data_scanner.rs b/crates/ecstore/src/heal/data_scanner.rs similarity index 100% rename from ecstore/src/heal/data_scanner.rs rename to crates/ecstore/src/heal/data_scanner.rs diff --git a/ecstore/src/heal/data_scanner_metric.rs b/crates/ecstore/src/heal/data_scanner_metric.rs similarity index 99% rename from ecstore/src/heal/data_scanner_metric.rs rename to crates/ecstore/src/heal/data_scanner_metric.rs index a69be875..8e0b8ccc 100644 --- a/ecstore/src/heal/data_scanner_metric.rs +++ b/crates/ecstore/src/heal/data_scanner_metric.rs @@ -12,10 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +use super::data_scanner::CurrentScannerCycle; +use crate::bucket::lifecycle::lifecycle; use chrono::Utc; -use common::last_minute::{AccElem, LastMinuteLatency}; use lazy_static::lazy_static; -use madmin::metrics::ScannerMetrics as M_ScannerMetrics; +use rustfs_common::last_minute::{AccElem, LastMinuteLatency}; +use rustfs_madmin::metrics::ScannerMetrics as M_ScannerMetrics; use std::{ collections::HashMap, pin::Pin, @@ -27,9 +29,6 @@ use std::{ }; use tokio::sync::{Mutex, RwLock}; -use super::data_scanner::CurrentScannerCycle; -use crate::bucket::lifecycle::lifecycle; - lazy_static! { pub static ref globalScannerMetrics: Arc = Arc::new(ScannerMetrics::new()); } diff --git a/ecstore/src/heal/data_usage.rs b/crates/ecstore/src/heal/data_usage.rs similarity index 100% rename from ecstore/src/heal/data_usage.rs rename to crates/ecstore/src/heal/data_usage.rs diff --git a/ecstore/src/heal/data_usage_cache.rs b/crates/ecstore/src/heal/data_usage_cache.rs similarity index 100% rename from ecstore/src/heal/data_usage_cache.rs rename to crates/ecstore/src/heal/data_usage_cache.rs diff --git a/ecstore/src/heal/error.rs b/crates/ecstore/src/heal/error.rs similarity index 100% rename from ecstore/src/heal/error.rs rename to crates/ecstore/src/heal/error.rs diff --git a/ecstore/src/heal/heal_commands.rs b/crates/ecstore/src/heal/heal_commands.rs similarity index 99% rename from ecstore/src/heal/heal_commands.rs rename to crates/ecstore/src/heal/heal_commands.rs index 37f9bf8b..3df0f3ba 100644 --- a/ecstore/src/heal/heal_commands.rs +++ b/crates/ecstore/src/heal/heal_commands.rs @@ -316,10 +316,10 @@ impl HealingTracker { }); } - pub async fn to_healing_disk(&self) -> madmin::HealingDisk { + pub async fn to_healing_disk(&self) -> rustfs_madmin::HealingDisk { let _ = self.mu.read().await; - madmin::HealingDisk { + rustfs_madmin::HealingDisk { id: self.id.clone(), heal_id: self.heal_id.clone(), pool_index: self.pool_index, @@ -461,7 +461,7 @@ pub struct SetStatus { pub heal_status: String, pub heal_priority: String, pub total_objects: usize, - pub disks: Vec, + pub disks: Vec, } #[derive(Debug, Default, Serialize, Deserialize)] diff --git a/ecstore/src/heal/heal_ops.rs b/crates/ecstore/src/heal/heal_ops.rs similarity index 99% rename from ecstore/src/heal/heal_ops.rs rename to crates/ecstore/src/heal/heal_ops.rs index b162d2d2..2ee4aee1 100644 --- a/ecstore/src/heal/heal_ops.rs +++ b/crates/ecstore/src/heal/heal_ops.rs @@ -37,8 +37,8 @@ use crate::{ use chrono::Utc; use futures::join; use lazy_static::lazy_static; -use madmin::heal_commands::{HealDriveInfo, HealItemType, HealResultItem}; use rustfs_filemeta::MetaCacheEntry; +use rustfs_madmin::heal_commands::{HealDriveInfo, HealItemType, HealResultItem}; use rustfs_utils::path::has_prefix; use rustfs_utils::path::path_join; use serde::{Deserialize, Serialize}; @@ -672,7 +672,7 @@ impl AllHealState { self.heal_status.write().await.insert(tracker.id.clone(), tracker.clone()); } - pub async fn get_local_healing_disks(&self) -> HashMap { + pub async fn get_local_healing_disks(&self) -> HashMap { let _ = self.mu.read().await; let mut dst = HashMap::new(); diff --git a/ecstore/src/heal/mod.rs b/crates/ecstore/src/heal/mod.rs similarity index 100% rename from ecstore/src/heal/mod.rs rename to crates/ecstore/src/heal/mod.rs diff --git a/ecstore/src/heal/mrf.rs b/crates/ecstore/src/heal/mrf.rs similarity index 100% rename from ecstore/src/heal/mrf.rs rename to crates/ecstore/src/heal/mrf.rs diff --git a/ecstore/src/lib.rs b/crates/ecstore/src/lib.rs similarity index 100% rename from ecstore/src/lib.rs rename to crates/ecstore/src/lib.rs diff --git a/ecstore/src/metrics_realtime.rs b/crates/ecstore/src/metrics_realtime.rs similarity index 98% rename from ecstore/src/metrics_realtime.rs rename to crates/ecstore/src/metrics_realtime.rs index 628e22a3..298ff846 100644 --- a/ecstore/src/metrics_realtime.rs +++ b/crates/ecstore/src/metrics_realtime.rs @@ -15,8 +15,8 @@ use std::collections::{HashMap, HashSet}; use chrono::Utc; -use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Addr}; -use madmin::metrics::{DiskIOStats, DiskMetric, RealtimeMetrics}; +use rustfs_common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Addr}; +use rustfs_madmin::metrics::{DiskIOStats, DiskMetric, RealtimeMetrics}; use rustfs_utils::os::get_drive_stats; use serde::{Deserialize, Serialize}; use tracing::info; diff --git a/ecstore/src/notification_sys.rs b/crates/ecstore/src/notification_sys.rs similarity index 96% rename from ecstore/src/notification_sys.rs rename to crates/ecstore/src/notification_sys.rs index 469b2545..7e629588 100644 --- a/ecstore/src/notification_sys.rs +++ b/crates/ecstore/src/notification_sys.rs @@ -20,7 +20,7 @@ use crate::rpc::PeerRestClient; use crate::{endpoints::EndpointServerPools, new_object_layer_fn}; use futures::future::join_all; use lazy_static::lazy_static; -use madmin::{ItemState, ServerProperties}; +use rustfs_madmin::{ItemState, ServerProperties}; use std::sync::OnceLock; use std::time::SystemTime; use tracing::{error, warn}; @@ -79,7 +79,7 @@ impl NotificationSys { unimplemented!() } - pub async fn storage_info(&self, api: &S) -> madmin::StorageInfo { + pub async fn storage_info(&self, api: &S) -> rustfs_madmin::StorageInfo { let mut futures = Vec::with_capacity(self.peer_clients.len()); for client in self.peer_clients.iter() { @@ -87,7 +87,7 @@ impl NotificationSys { if let Some(client) = client { match client.local_storage_info().await { Ok(info) => Some(info), - Err(_) => Some(madmin::StorageInfo { + Err(_) => Some(rustfs_madmin::StorageInfo { disks: get_offline_disks(&client.host.to_string(), &get_global_endpoints()), ..Default::default() }), @@ -108,7 +108,7 @@ impl NotificationSys { } let backend = api.backend_info().await; - madmin::StorageInfo { disks, backend } + rustfs_madmin::StorageInfo { disks, backend } } pub async fn server_info(&self) -> Vec { @@ -204,13 +204,13 @@ impl NotificationSys { } } -fn get_offline_disks(offline_host: &str, endpoints: &EndpointServerPools) -> Vec { +fn get_offline_disks(offline_host: &str, endpoints: &EndpointServerPools) -> Vec { let mut offline_disks = Vec::new(); for pool in endpoints.as_ref() { for ep in pool.endpoints.as_ref() { if (offline_host.is_empty() && ep.is_local) || offline_host == ep.host_port() { - offline_disks.push(madmin::Disk { + offline_disks.push(rustfs_madmin::Disk { endpoint: ep.to_string(), state: ItemState::Offline.to_string().to_owned(), pool_index: ep.pool_idx, diff --git a/ecstore/src/pools.rs b/crates/ecstore/src/pools.rs similarity index 99% rename from ecstore/src/pools.rs rename to crates/ecstore/src/pools.rs index 1eb1640d..aa4f3741 100644 --- a/ecstore/src/pools.rs +++ b/crates/ecstore/src/pools.rs @@ -31,15 +31,15 @@ use crate::store_api::{ BucketOptions, CompletePart, GetObjectReader, MakeBucketOptions, ObjectIO, ObjectOptions, PutObjReader, StorageAPI, }; use crate::{sets::Sets, store::ECStore}; -use ::workers::workers::Workers; use byteorder::{ByteOrder, LittleEndian, WriteBytesExt}; -use common::defer; use futures::future::BoxFuture; use http::HeaderMap; use rmp_serde::{Deserializer, Serializer}; +use rustfs_common::defer; use rustfs_filemeta::{MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_rio::{HashReader, WarpReader}; use rustfs_utils::path::{SLASH_SEPARATOR, encode_dir_object, path_join}; +use rustfs_workers::workers::Workers; use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::fmt::Display; @@ -1383,7 +1383,7 @@ impl SetDisks { } } -pub fn get_total_usable_capacity(disks: &[madmin::Disk], info: &madmin::StorageInfo) -> usize { +pub fn get_total_usable_capacity(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize { let mut capacity = 0; for disk in disks.iter() { if disk.pool_index < 0 || info.backend.standard_sc_data.len() <= disk.pool_index as usize { @@ -1396,7 +1396,7 @@ pub fn get_total_usable_capacity(disks: &[madmin::Disk], info: &madmin::StorageI capacity } -pub fn get_total_usable_capacity_free(disks: &[madmin::Disk], info: &madmin::StorageInfo) -> usize { +pub fn get_total_usable_capacity_free(disks: &[rustfs_madmin::Disk], info: &rustfs_madmin::StorageInfo) -> usize { let mut capacity = 0; for disk in disks.iter() { if disk.pool_index < 0 || info.backend.standard_sc_data.len() <= disk.pool_index as usize { diff --git a/ecstore/src/rebalance.rs b/crates/ecstore/src/rebalance.rs similarity index 99% rename from ecstore/src/rebalance.rs rename to crates/ecstore/src/rebalance.rs index 1d05dc69..5ab2f149 100644 --- a/ecstore/src/rebalance.rs +++ b/crates/ecstore/src/rebalance.rs @@ -23,12 +23,13 @@ use crate::pools::ListCallback; use crate::set_disk::SetDisks; use crate::store::ECStore; use crate::store_api::{CompletePart, GetObjectReader, ObjectIO, ObjectOptions, PutObjReader}; -use common::defer; use http::HeaderMap; +use rustfs_common::defer; use rustfs_filemeta::{FileInfo, MetaCacheEntries, MetaCacheEntry, MetadataResolutionParams}; use rustfs_rio::{HashReader, WarpReader}; use rustfs_utils::path::encode_dir_object; use serde::{Deserialize, Serialize}; +use std::fmt; use std::io::Cursor; use std::sync::Arc; use time::OffsetDateTime; @@ -98,8 +99,6 @@ pub enum RebalStatus { Failed, } -use std::fmt; - impl fmt::Display for RebalStatus { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let status = match self { diff --git a/ecstore/src/rpc/http_auth.rs b/crates/ecstore/src/rpc/http_auth.rs similarity index 100% rename from ecstore/src/rpc/http_auth.rs rename to crates/ecstore/src/rpc/http_auth.rs diff --git a/ecstore/src/rpc/mod.rs b/crates/ecstore/src/rpc/mod.rs similarity index 100% rename from ecstore/src/rpc/mod.rs rename to crates/ecstore/src/rpc/mod.rs diff --git a/ecstore/src/rpc/peer_rest_client.rs b/crates/ecstore/src/rpc/peer_rest_client.rs similarity index 99% rename from ecstore/src/rpc/peer_rest_client.rs rename to crates/ecstore/src/rpc/peer_rest_client.rs index 4dabf4a6..54aafa60 100644 --- a/ecstore/src/rpc/peer_rest_client.rs +++ b/crates/ecstore/src/rpc/peer_rest_client.rs @@ -19,13 +19,14 @@ use crate::{ heal::heal_commands::BgHealState, metrics_realtime::{CollectMetricsOpts, MetricType}, }; -use madmin::{ +use rmp_serde::{Deserializer, Serializer}; +use rustfs_madmin::{ ServerProperties, health::{Cpus, MemInfo, OsInfo, Partitions, ProcInfo, SysConfig, SysErrors, SysService}, metrics::RealtimeMetrics, net::NetInfo, }; -use protos::{ +use rustfs_protos::{ node_service_time_out_client, proto_gen::node_service::{ BackgroundHealStatusRequest, DeleteBucketMetadataRequest, DeletePolicyRequest, DeleteServiceAccountRequest, @@ -37,7 +38,6 @@ use protos::{ StartProfilingRequest, StopRebalanceRequest, }, }; -use rmp_serde::{Deserializer, Serializer}; use rustfs_utils::XHost; use serde::{Deserialize, Serialize as _}; use std::{collections::HashMap, io::Cursor, time::SystemTime}; @@ -87,7 +87,7 @@ impl PeerRestClient { } impl PeerRestClient { - pub async fn local_storage_info(&self) -> Result { + pub async fn local_storage_info(&self) -> Result { let mut client = node_service_time_out_client(&self.grid_host) .await .map_err(|err| Error::other(err.to_string()))?; @@ -103,7 +103,7 @@ impl PeerRestClient { let data = response.storage_info; let mut buf = Deserializer::new(Cursor::new(data)); - let storage_info: madmin::StorageInfo = Deserialize::deserialize(&mut buf)?; + let storage_info: rustfs_madmin::StorageInfo = Deserialize::deserialize(&mut buf)?; Ok(storage_info) } diff --git a/ecstore/src/rpc/peer_s3_client.rs b/crates/ecstore/src/rpc/peer_s3_client.rs similarity index 99% rename from ecstore/src/rpc/peer_s3_client.rs rename to crates/ecstore/src/rpc/peer_s3_client.rs index 9edea5c2..2da4e244 100644 --- a/ecstore/src/rpc/peer_s3_client.rs +++ b/crates/ecstore/src/rpc/peer_s3_client.rs @@ -30,9 +30,9 @@ use crate::{ }; use async_trait::async_trait; use futures::future::join_all; -use madmin::heal_commands::{HealDriveInfo, HealResultItem}; -use protos::node_service_time_out_client; -use protos::proto_gen::node_service::{ +use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem}; +use rustfs_protos::node_service_time_out_client; +use rustfs_protos::proto_gen::node_service::{ DeleteBucketRequest, GetBucketInfoRequest, HealBucketRequest, ListBucketRequest, MakeBucketRequest, }; use std::{collections::HashMap, fmt::Debug, sync::Arc}; diff --git a/ecstore/src/rpc/remote_disk.rs b/crates/ecstore/src/rpc/remote_disk.rs similarity index 99% rename from ecstore/src/rpc/remote_disk.rs rename to crates/ecstore/src/rpc/remote_disk.rs index 3aa2b1b3..161c3a76 100644 --- a/ecstore/src/rpc/remote_disk.rs +++ b/crates/ecstore/src/rpc/remote_disk.rs @@ -17,7 +17,7 @@ use std::path::PathBuf; use bytes::Bytes; use futures::lock::Mutex; use http::{HeaderMap, HeaderValue, Method, header::CONTENT_TYPE}; -use protos::{ +use rustfs_protos::{ node_service_time_out_client, proto_gen::node_service::{ CheckPartsRequest, DeletePathsRequest, DeleteRequest, DeleteVersionRequest, DeleteVersionsRequest, DeleteVolumeRequest, @@ -45,6 +45,7 @@ use crate::{ }, }; use rustfs_filemeta::{FileInfo, RawFileInfo}; +use rustfs_protos::proto_gen::node_service::RenamePartRequest; use rustfs_rio::{HttpReader, HttpWriter}; use tokio::{ io::AsyncWrite, @@ -55,8 +56,6 @@ use tonic::Request; use tracing::info; use uuid::Uuid; -use protos::proto_gen::node_service::RenamePartRequest; - #[derive(Debug)] pub struct RemoteDisk { pub id: Mutex>, diff --git a/ecstore/src/rpc/tonic_service.rs b/crates/ecstore/src/rpc/tonic_service.rs similarity index 97% rename from ecstore/src/rpc/tonic_service.rs rename to crates/ecstore/src/rpc/tonic_service.rs index 9e883a34..faaa3726 100644 --- a/ecstore/src/rpc/tonic_service.rs +++ b/crates/ecstore/src/rpc/tonic_service.rs @@ -34,21 +34,21 @@ use crate::{ }; use futures::{Stream, StreamExt}; use futures_util::future::join_all; -use lock::{GLOBAL_LOCAL_SERVER, Locker, lock_args::LockArgs}; +use rustfs_lock::{GLOBAL_LOCAL_SERVER, Locker, lock_args::LockArgs}; -use common::globals::GLOBAL_Local_Node_Name; +use rustfs_common::globals::GLOBAL_Local_Node_Name; use bytes::Bytes; -use madmin::health::{ +use rmp_serde::{Deserializer, Serializer}; +use rustfs_filemeta::{FileInfo, MetacacheReader}; +use rustfs_madmin::health::{ get_cpus, get_mem_info, get_os_info, get_partitions, get_proc_info, get_sys_config, get_sys_errors, get_sys_services, }; -use madmin::net::get_net_info; -use protos::{ +use rustfs_madmin::net::get_net_info; +use rustfs_protos::{ models::{PingBody, PingBodyBuilder}, proto_gen::node_service::{node_service_server::NodeService as Node, *}, }; -use rmp_serde::{Deserializer, Serializer}; -use rustfs_filemeta::{FileInfo, MetacacheReader}; use serde::{Deserialize, Serialize}; use tokio::spawn; use tokio::sync::mpsc; @@ -2246,31 +2246,19 @@ impl Node for NodeService { #[allow(unused_imports)] mod tests { use super::*; - use protos::proto_gen::node_service::{ - BackgroundHealStatusRequest, BackgroundHealStatusResponse, CheckPartsRequest, CheckPartsResponse, - DeleteBucketMetadataRequest, DeleteBucketMetadataResponse, DeleteBucketRequest, DeleteBucketResponse, DeletePathsRequest, - DeletePathsResponse, DeletePolicyRequest, DeletePolicyResponse, DeleteRequest, DeleteResponse, - DeleteServiceAccountRequest, DeleteServiceAccountResponse, DeleteUserRequest, DeleteUserResponse, DeleteVersionRequest, - DeleteVersionResponse, DeleteVersionsRequest, DeleteVersionsResponse, DeleteVolumeRequest, DeleteVolumeResponse, - DiskInfoRequest, DiskInfoResponse, GenerallyLockRequest, GenerallyLockResponse, GetBucketInfoRequest, - GetBucketInfoResponse, GetCpusRequest, GetCpusResponse, GetMemInfoRequest, GetMemInfoResponse, GetNetInfoRequest, - GetNetInfoResponse, GetOsInfoRequest, GetOsInfoResponse, GetPartitionsRequest, GetPartitionsResponse, GetProcInfoRequest, - GetProcInfoResponse, GetSeLinuxInfoRequest, GetSeLinuxInfoResponse, GetSysConfigRequest, GetSysConfigResponse, - GetSysErrorsRequest, GetSysErrorsResponse, HealBucketRequest, HealBucketResponse, ListBucketRequest, ListBucketResponse, - ListDirRequest, ListDirResponse, ListVolumesRequest, ListVolumesResponse, LoadBucketMetadataRequest, - LoadBucketMetadataResponse, LoadGroupRequest, LoadGroupResponse, LoadPolicyMappingRequest, LoadPolicyMappingResponse, - LoadPolicyRequest, LoadPolicyResponse, LoadRebalanceMetaRequest, LoadRebalanceMetaResponse, LoadServiceAccountRequest, - LoadServiceAccountResponse, LoadUserRequest, LoadUserResponse, LocalStorageInfoRequest, LocalStorageInfoResponse, - MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, - Mss, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadMultipleRequest, ReadMultipleResponse, - ReadVersionRequest, ReadVersionResponse, ReadXlRequest, ReadXlResponse, ReloadPoolMetaRequest, ReloadPoolMetaResponse, - ReloadSiteReplicationConfigRequest, ReloadSiteReplicationConfigResponse, RenameDataRequest, RenameDataResponse, - RenameFileRequest, RenameFileResponse, RenamePartRequest, RenamePartResponse, ServerInfoRequest, ServerInfoResponse, - SignalServiceRequest, SignalServiceResponse, StatVolumeRequest, StatVolumeResponse, StopRebalanceRequest, - StopRebalanceResponse, UpdateMetadataRequest, UpdateMetadataResponse, VerifyFileRequest, VerifyFileResponse, - WriteAllRequest, WriteAllResponse, WriteMetadataRequest, WriteMetadataResponse, + use rustfs_protos::proto_gen::node_service::{ + BackgroundHealStatusRequest, CheckPartsRequest, DeleteBucketMetadataRequest, DeleteBucketRequest, DeletePathsRequest, + DeletePolicyRequest, DeleteRequest, DeleteServiceAccountRequest, DeleteUserRequest, DeleteVersionRequest, + DeleteVersionsRequest, DeleteVolumeRequest, DiskInfoRequest, GenerallyLockRequest, GetBucketInfoRequest, GetCpusRequest, + GetMemInfoRequest, GetNetInfoRequest, GetOsInfoRequest, GetPartitionsRequest, GetProcInfoRequest, GetSeLinuxInfoRequest, + GetSysConfigRequest, GetSysErrorsRequest, HealBucketRequest, ListBucketRequest, ListDirRequest, ListVolumesRequest, + LoadBucketMetadataRequest, LoadGroupRequest, LoadPolicyMappingRequest, LoadPolicyRequest, LoadRebalanceMetaRequest, + LoadServiceAccountRequest, LoadUserRequest, LocalStorageInfoRequest, MakeBucketRequest, MakeVolumeRequest, + MakeVolumesRequest, PingRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, + ReloadPoolMetaRequest, ReloadSiteReplicationConfigRequest, RenameDataRequest, RenameFileRequest, RenamePartRequest, + ServerInfoRequest, StatVolumeRequest, StopRebalanceRequest, UpdateMetadataRequest, VerifyFileRequest, WriteAllRequest, + WriteMetadataRequest, }; - use std::collections::HashMap; use tonic::Request; fn create_test_node_service() -> NodeService { diff --git a/ecstore/src/set_disk.rs b/crates/ecstore/src/set_disk.rs similarity index 99% rename from ecstore/src/set_disk.rs rename to crates/ecstore/src/set_disk.rs index 1d2f07d5..a0b61907 100644 --- a/ecstore/src/set_disk.rs +++ b/crates/ecstore/src/set_disk.rs @@ -74,8 +74,6 @@ use chrono::Utc; use futures::future::join_all; use glob::Pattern; use http::HeaderMap; -use lock::{LockApi, namespace_lock::NsLockMap}; -use madmin::heal_commands::{HealDriveInfo, HealResultItem}; use md5::{Digest as Md5Digest, Md5}; use rand::{Rng, seq::SliceRandom}; use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER; @@ -85,12 +83,15 @@ use rustfs_filemeta::{ headers::{AMZ_OBJECT_TAGGING, AMZ_STORAGE_CLASS}, merge_file_meta_versions, }; +use rustfs_lock::{LockApi, namespace_lock::NsLockMap}; +use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem}; use rustfs_rio::{EtagResolvable, HashReader, TryGetIndex as _, WarpReader}; use rustfs_utils::{ HashAlgorithm, crypto::{base64_decode, base64_encode, hex}, path::{SLASH_SEPARATOR, encode_dir_object, has_suffix, path_join_buf}, }; +use rustfs_workers::workers::Workers; use s3s::header::X_AMZ_RESTORE; use sha2::{Digest, Sha256}; use std::hash::Hash; @@ -116,7 +117,6 @@ use tokio::{ use tracing::error; use tracing::{debug, info, warn}; use uuid::Uuid; -use workers::workers::Workers; pub const DEFAULT_READ_BUFFER_SIZE: usize = 1024 * 1024; @@ -3306,7 +3306,7 @@ impl SetDisks { num_healers ); - let jt = Workers::new(num_healers).map_err(|err| Error::other(err.to_string()))?; + let jt = rustfs_workers::workers::Workers::new(num_healers).map_err(|err| Error::other(err.to_string()))?; let heal_entry_done = |name: String| HealEntryResult { entry_done: true, @@ -4140,17 +4140,17 @@ impl ObjectIO for SetDisks { #[async_trait::async_trait] impl StorageAPI for SetDisks { #[tracing::instrument(skip(self))] - async fn backend_info(&self) -> madmin::BackendInfo { + async fn backend_info(&self) -> rustfs_madmin::BackendInfo { unimplemented!() } #[tracing::instrument(skip(self))] - async fn storage_info(&self) -> madmin::StorageInfo { + async fn storage_info(&self) -> rustfs_madmin::StorageInfo { let disks = self.get_disks_internal().await; get_storage_info(&disks, &self.set_endpoints).await } #[tracing::instrument(skip(self))] - async fn local_storage_info(&self) -> madmin::StorageInfo { + async fn local_storage_info(&self) -> rustfs_madmin::StorageInfo { let disks = self.get_disks_internal().await; let mut local_disks: Vec>> = Vec::new(); @@ -6072,13 +6072,13 @@ pub fn should_heal_object_on_disk( (false, err.clone()) } -async fn get_disks_info(disks: &[Option], eps: &[Endpoint]) -> Vec { +async fn get_disks_info(disks: &[Option], eps: &[Endpoint]) -> Vec { let mut ret = Vec::new(); for (i, pool) in disks.iter().enumerate() { if let Some(disk) = pool { match disk.disk_info(&DiskInfoOptions::default()).await { - Ok(res) => ret.push(madmin::Disk { + Ok(res) => ret.push(rustfs_madmin::Disk { endpoint: eps[i].to_string(), local: eps[i].is_local, pool_index: eps[i].pool_idx, @@ -6109,7 +6109,7 @@ async fn get_disks_info(disks: &[Option], eps: &[Endpoint]) -> Vec ret.push(madmin::Disk { + Err(err) => ret.push(rustfs_madmin::Disk { state: err.to_string(), endpoint: eps[i].to_string(), local: eps[i].is_local, @@ -6120,7 +6120,7 @@ async fn get_disks_info(disks: &[Option], eps: &[Endpoint]) -> Vec], eps: &[Endpoint]) -> Vec], eps: &[Endpoint]) -> madmin::StorageInfo { +async fn get_storage_info(disks: &[Option], eps: &[Endpoint]) -> rustfs_madmin::StorageInfo { let mut disks = get_disks_info(disks, eps).await; disks.sort_by(|a, b| a.total_space.cmp(&b.total_space)); - madmin::StorageInfo { + rustfs_madmin::StorageInfo { disks, - backend: madmin::BackendInfo { - backend_type: madmin::BackendByte::Erasure, + backend: rustfs_madmin::BackendInfo { + backend_type: rustfs_madmin::BackendByte::Erasure, ..Default::default() }, } @@ -6214,7 +6214,7 @@ mod tests { // Test that all CHECK_PART constants have expected values assert_eq!(CHECK_PART_UNKNOWN, 0); assert_eq!(CHECK_PART_SUCCESS, 1); - assert_eq!(CHECK_PART_FILE_NOT_FOUND, 4); // 实际值是4，不是2 + assert_eq!(CHECK_PART_FILE_NOT_FOUND, 4); // 实际值是 4，不是 2 assert_eq!(CHECK_PART_VOLUME_NOT_FOUND, 3); assert_eq!(CHECK_PART_FILE_CORRUPT, 5); } @@ -6479,7 +6479,7 @@ mod tests { assert_eq!(conv_part_err_to_int(&Some(disk_err)), CHECK_PART_FILE_NOT_FOUND); let other_err = DiskError::other("other error"); - assert_eq!(conv_part_err_to_int(&Some(other_err)), CHECK_PART_UNKNOWN); // other错误应该返回UNKNOWN，不是SUCCESS + assert_eq!(conv_part_err_to_int(&Some(other_err)), CHECK_PART_UNKNOWN); // other 错误应该返回 UNKNOWN，不是 SUCCESS } #[test] diff --git a/ecstore/src/sets.rs b/crates/ecstore/src/sets.rs similarity index 98% rename from ecstore/src/sets.rs rename to crates/ecstore/src/sets.rs index f5074454..28109afc 100644 --- a/ecstore/src/sets.rs +++ b/crates/ecstore/src/sets.rs @@ -38,12 +38,12 @@ use crate::{ }, store_init::{check_format_erasure_values, get_format_erasure_in_quorum, load_format_erasure_all, save_format_file}, }; -use common::globals::GLOBAL_Local_Node_Name; use futures::future::join_all; use http::HeaderMap; -use lock::{LockApi, namespace_lock::NsLockMap, new_lock_api}; -use madmin::heal_commands::{HealDriveInfo, HealResultItem}; +use rustfs_common::globals::GLOBAL_Local_Node_Name; use rustfs_filemeta::FileInfo; +use rustfs_lock::{LockApi, namespace_lock::NsLockMap, new_lock_api}; +use rustfs_madmin::heal_commands::{HealDriveInfo, HealResultItem}; use rustfs_utils::{crc_hash, path::path_join_buf, sip_hash}; use tokio::sync::RwLock; use uuid::Uuid; @@ -367,11 +367,11 @@ impl ObjectIO for Sets { #[async_trait::async_trait] impl StorageAPI for Sets { #[tracing::instrument(skip(self))] - async fn backend_info(&self) -> madmin::BackendInfo { + async fn backend_info(&self) -> rustfs_madmin::BackendInfo { unimplemented!() } #[tracing::instrument(skip(self))] - async fn storage_info(&self) -> madmin::StorageInfo { + async fn storage_info(&self) -> rustfs_madmin::StorageInfo { let mut futures = Vec::with_capacity(self.disk_set.len()); for set in self.disk_set.iter() { @@ -386,13 +386,13 @@ impl StorageAPI for Sets { disks.extend_from_slice(&res.disks); } - madmin::StorageInfo { + rustfs_madmin::StorageInfo { disks, ..Default::default() } } #[tracing::instrument(skip(self))] - async fn local_storage_info(&self) -> madmin::StorageInfo { + async fn local_storage_info(&self) -> rustfs_madmin::StorageInfo { let mut futures = Vec::with_capacity(self.disk_set.len()); for set in self.disk_set.iter() { @@ -406,7 +406,7 @@ impl StorageAPI for Sets { for res in results.into_iter() { disks.extend_from_slice(&res.disks); } - madmin::StorageInfo { + rustfs_madmin::StorageInfo { disks, ..Default::default() } diff --git a/ecstore/src/store.rs b/crates/ecstore/src/store.rs similarity index 99% rename from ecstore/src/store.rs rename to crates/ecstore/src/store.rs index b30d16e8..bff22f7d 100644 --- a/ecstore/src/store.rs +++ b/crates/ecstore/src/store.rs @@ -53,15 +53,15 @@ use crate::{ }, store_init, }; -use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port}; use futures::future::join_all; use glob::Pattern; use http::HeaderMap; use lazy_static::lazy_static; -use madmin::heal_commands::HealResultItem; use rand::Rng as _; +use rustfs_common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Host, GLOBAL_Rustfs_Port}; use rustfs_filemeta::FileInfo; use rustfs_filemeta::MetaCacheEntry; +use rustfs_madmin::heal_commands::HealResultItem; use rustfs_utils::crypto::base64_decode; use rustfs_utils::path::{SLASH_SEPARATOR, decode_dir_object, encode_dir_object, path_join_buf}; use s3s::dto::{BucketVersioningStatus, ObjectLockConfiguration, ObjectLockEnabled, VersioningConfiguration}; @@ -875,7 +875,7 @@ impl ECStore { let (update_closer_tx, mut update_close_rx) = mpsc::channel(10); let mut ctx_clone = cancel.subscribe(); let all_buckets_clone = all_buckets.clone(); - // 新增：从环境变量读取interval，默认30秒 + // 新增：从环境变量读取 interval，默认 30 秒 let ns_scanner_interval_secs = std::env::var("RUSTFS_NS_SCANNER_INTERVAL") .ok() .and_then(|v| v.parse::().ok()) @@ -888,7 +888,7 @@ impl ECStore { .unwrap_or(false); if skip_background_task { - info!("跳过后台任务执行: RUSTFS_SKIP_BACKGROUND_TASK=true"); + info!("跳过后台任务执行：RUSTFS_SKIP_BACKGROUND_TASK=true"); return Ok(()); } @@ -1289,7 +1289,7 @@ lazy_static! { #[async_trait::async_trait] impl StorageAPI for ECStore { #[tracing::instrument(skip(self))] - async fn backend_info(&self) -> madmin::BackendInfo { + async fn backend_info(&self) -> rustfs_madmin::BackendInfo { let (standard_sc_parity, rr_sc_parity) = { if let Some(sc) = GLOBAL_StorageClass.get() { let sc_parity = sc @@ -1320,10 +1320,10 @@ impl StorageAPI for ECStore { drives_per_set.push(*set_count); } - madmin::BackendInfo { - backend_type: madmin::BackendByte::Erasure, - online_disks: madmin::BackendDisks::new(), - offline_disks: madmin::BackendDisks::new(), + rustfs_madmin::BackendInfo { + backend_type: rustfs_madmin::BackendByte::Erasure, + online_disks: rustfs_madmin::BackendDisks::new(), + offline_disks: rustfs_madmin::BackendDisks::new(), standard_sc_data, standard_sc_parity, rr_sc_data, @@ -1334,15 +1334,15 @@ impl StorageAPI for ECStore { } } #[tracing::instrument(skip(self))] - async fn storage_info(&self) -> madmin::StorageInfo { + async fn storage_info(&self) -> rustfs_madmin::StorageInfo { let Some(notification_sy) = get_global_notification_sys() else { - return madmin::StorageInfo::default(); + return rustfs_madmin::StorageInfo::default(); }; notification_sy.storage_info(self).await } #[tracing::instrument(skip(self))] - async fn local_storage_info(&self) -> madmin::StorageInfo { + async fn local_storage_info(&self) -> rustfs_madmin::StorageInfo { let mut futures = Vec::with_capacity(self.pools.len()); for pool in self.pools.iter() { @@ -1358,7 +1358,7 @@ impl StorageAPI for ECStore { } let backend = self.backend_info().await; - madmin::StorageInfo { backend, disks } + rustfs_madmin::StorageInfo { backend, disks } } #[tracing::instrument(skip(self))] diff --git a/ecstore/src/store_api.rs b/crates/ecstore/src/store_api.rs similarity index 99% rename from ecstore/src/store_api.rs rename to crates/ecstore/src/store_api.rs index 64ede69d..a5f2add9 100644 --- a/ecstore/src/store_api.rs +++ b/crates/ecstore/src/store_api.rs @@ -25,9 +25,9 @@ use crate::{ }; use crate::{disk::DiskStore, heal::heal_commands::HealOpts}; use http::{HeaderMap, HeaderValue}; -use madmin::heal_commands::HealResultItem; use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER; use rustfs_filemeta::{FileInfo, MetaCacheEntriesSorted, ObjectPartInfo, headers::AMZ_OBJECT_TAGGING}; +use rustfs_madmin::heal_commands::HealResultItem; use rustfs_rio::{DecompressReader, HashReader, LimitReader, WarpReader}; use rustfs_utils::CompressionAlgorithm; use rustfs_utils::path::decode_dir_object; @@ -892,9 +892,9 @@ pub trait StorageAPI: ObjectIO { // Shutdown TODO: // NSScanner TODO: - async fn backend_info(&self) -> madmin::BackendInfo; - async fn storage_info(&self) -> madmin::StorageInfo; - async fn local_storage_info(&self) -> madmin::StorageInfo; + async fn backend_info(&self) -> rustfs_madmin::BackendInfo; + async fn storage_info(&self) -> rustfs_madmin::StorageInfo; + async fn local_storage_info(&self) -> rustfs_madmin::StorageInfo; async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()>; async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result; diff --git a/ecstore/src/store_init.rs b/crates/ecstore/src/store_init.rs similarity index 100% rename from ecstore/src/store_init.rs rename to crates/ecstore/src/store_init.rs diff --git a/ecstore/src/store_list_objects.rs b/crates/ecstore/src/store_list_objects.rs similarity index 99% rename from ecstore/src/store_list_objects.rs rename to crates/ecstore/src/store_list_objects.rs index b7b88f35..fa6c4d7f 100644 --- a/ecstore/src/store_list_objects.rs +++ b/crates/ecstore/src/store_list_objects.rs @@ -1406,7 +1406,7 @@ mod test { // use crate::store_list_objects::WalkOptions; // use crate::store_list_objects::WalkVersionsSortOrder; // use futures::future::join_all; - // use lock::namespace_lock::NsLockMap; + // use rustfs_lock::namespace_lock::NsLockMap; // use tokio::sync::broadcast; // use tokio::sync::mpsc; // use tokio::sync::RwLock; diff --git a/ecstore/src/store_utils.rs b/crates/ecstore/src/store_utils.rs similarity index 100% rename from ecstore/src/store_utils.rs rename to crates/ecstore/src/store_utils.rs diff --git a/ecstore/src/tier/mod.rs b/crates/ecstore/src/tier/mod.rs similarity index 100% rename from ecstore/src/tier/mod.rs rename to crates/ecstore/src/tier/mod.rs diff --git a/ecstore/src/tier/tier.rs b/crates/ecstore/src/tier/tier.rs similarity index 100% rename from ecstore/src/tier/tier.rs rename to crates/ecstore/src/tier/tier.rs diff --git a/ecstore/src/tier/tier_admin.rs b/crates/ecstore/src/tier/tier_admin.rs similarity index 100% rename from ecstore/src/tier/tier_admin.rs rename to crates/ecstore/src/tier/tier_admin.rs diff --git a/ecstore/src/tier/tier_config.rs b/crates/ecstore/src/tier/tier_config.rs similarity index 100% rename from ecstore/src/tier/tier_config.rs rename to crates/ecstore/src/tier/tier_config.rs diff --git a/ecstore/src/tier/tier_gen.rs b/crates/ecstore/src/tier/tier_gen.rs similarity index 100% rename from ecstore/src/tier/tier_gen.rs rename to crates/ecstore/src/tier/tier_gen.rs diff --git a/ecstore/src/tier/tier_handlers.rs b/crates/ecstore/src/tier/tier_handlers.rs similarity index 100% rename from ecstore/src/tier/tier_handlers.rs rename to crates/ecstore/src/tier/tier_handlers.rs diff --git a/ecstore/src/tier/warm_backend.rs b/crates/ecstore/src/tier/warm_backend.rs similarity index 100% rename from ecstore/src/tier/warm_backend.rs rename to crates/ecstore/src/tier/warm_backend.rs diff --git a/ecstore/src/tier/warm_backend_minio.rs b/crates/ecstore/src/tier/warm_backend_minio.rs similarity index 100% rename from ecstore/src/tier/warm_backend_minio.rs rename to crates/ecstore/src/tier/warm_backend_minio.rs diff --git a/ecstore/src/tier/warm_backend_rustfs.rs b/crates/ecstore/src/tier/warm_backend_rustfs.rs similarity index 100% rename from ecstore/src/tier/warm_backend_rustfs.rs rename to crates/ecstore/src/tier/warm_backend_rustfs.rs diff --git a/ecstore/src/tier/warm_backend_s3.rs b/crates/ecstore/src/tier/warm_backend_s3.rs similarity index 100% rename from ecstore/src/tier/warm_backend_s3.rs rename to crates/ecstore/src/tier/warm_backend_s3.rs diff --git a/crates/filemeta/Cargo.toml b/crates/filemeta/Cargo.toml index b96d2274..ef2d96a5 100644 --- a/crates/filemeta/Cargo.toml +++ b/crates/filemeta/Cargo.toml @@ -21,24 +21,23 @@ rust-version.workspace = true version.workspace = true [dependencies] -crc32fast = "1.4.2" +crc32fast = { workspace = true } rmp.workspace = true rmp-serde.workspace = true serde.workspace = true time.workspace = true uuid = { workspace = true, features = ["v4", "fast-rng", "serde"] } tokio = { workspace = true, features = ["io-util", "macros", "sync"] } -xxhash-rust = { version = "0.8.15", features = ["xxh64"] } +xxhash-rust = { workspace = true, features = ["xxh64"] } bytes.workspace = true -rustfs-utils = {workspace = true, features= ["hash"]} -byteorder = "1.5.0" +rustfs-utils = { workspace = true, features = ["hash"] } +byteorder = { workspace = true } tracing.workspace = true thiserror.workspace = true - s3s.workspace = true [dev-dependencies] -criterion = { version = "0.5", features = ["html_reports"] } +criterion = { workspace = true } [[bench]] name = "xl_meta_bench" diff --git a/iam/Cargo.toml b/crates/iam/Cargo.toml similarity index 78% rename from iam/Cargo.toml rename to crates/iam/Cargo.toml index 1813afc7..0dcf7077 100644 --- a/iam/Cargo.toml +++ b/crates/iam/Cargo.toml @@ -13,7 +13,7 @@ # limitations under the License. [package] -name = "iam" +name = "rustfs-iam" edition.workspace = true license.workspace = true repository.workspace = true @@ -27,26 +27,20 @@ workspace = true tokio.workspace = true time = { workspace = true, features = ["serde-human-readable"] } serde = { workspace = true, features = ["derive", "rc"] } -ecstore = { path = "../ecstore" } -policy.workspace = true +rustfs-ecstore = { workspace = true } +rustfs-policy.workspace = true serde_json.workspace = true async-trait.workspace = true thiserror.workspace = true -strum = { workspace = true, features = ["derive"] } arc-swap = { workspace = true } -crypto = { path = "../crypto" } -ipnetwork = { workspace = true, features = ["serde"] } -itertools = { workspace = true } +rustfs-crypto = { workspace = true } futures.workspace = true rand.workspace = true base64-simd = { workspace = true } jsonwebtoken = { workspace = true } tracing.workspace = true -madmin.workspace = true +rustfs-madmin.workspace = true lazy_static.workspace = true -regex = { workspace = true } -common.workspace = true rustfs-utils = { workspace = true, features = ["path"] } [dev-dependencies] -test-case.workspace = true diff --git a/iam/src/cache.rs b/crates/iam/src/cache.rs similarity index 99% rename from iam/src/cache.rs rename to crates/iam/src/cache.rs index dd5fd7b8..22fc215b 100644 --- a/iam/src/cache.rs +++ b/crates/iam/src/cache.rs @@ -20,7 +20,7 @@ use std::{ }; use arc_swap::{ArcSwap, AsRaw, Guard}; -use policy::{ +use rustfs_policy::{ auth::UserIdentity, policy::{Args, PolicyDoc}, }; diff --git a/iam/src/error.rs b/crates/iam/src/error.rs similarity index 78% rename from iam/src/error.rs rename to crates/iam/src/error.rs index 94a47d47..2a654c43 100644 --- a/iam/src/error.rs +++ b/crates/iam/src/error.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use policy::policy::Error as PolicyError; +use rustfs_policy::policy::Error as PolicyError; pub type Result = core::result::Result; @@ -25,7 +25,7 @@ pub enum Error { StringError(String), #[error("crypto: {0}")] - CryptoError(#[from] crypto::Error), + CryptoError(#[from] rustfs_crypto::Error), #[error("user '{0}' does not exist")] NoSuchUser(String), @@ -175,57 +175,57 @@ impl Error { } } -impl From for Error { - fn from(e: ecstore::error::StorageError) -> Self { +impl From for Error { + fn from(e: rustfs_ecstore::error::StorageError) -> Self { match e { - ecstore::error::StorageError::ConfigNotFound => Error::ConfigNotFound, + rustfs_ecstore::error::StorageError::ConfigNotFound => Error::ConfigNotFound, _ => Error::other(e), } } } -impl From for ecstore::error::StorageError { +impl From for rustfs_ecstore::error::StorageError { fn from(e: Error) -> Self { match e { - Error::ConfigNotFound => ecstore::error::StorageError::ConfigNotFound, - _ => ecstore::error::StorageError::other(e), + Error::ConfigNotFound => rustfs_ecstore::error::StorageError::ConfigNotFound, + _ => rustfs_ecstore::error::StorageError::other(e), } } } -impl From for Error { - fn from(e: policy::error::Error) -> Self { +impl From for Error { + fn from(e: rustfs_policy::error::Error) -> Self { match e { - policy::error::Error::PolicyTooLarge => Error::PolicyTooLarge, - policy::error::Error::InvalidArgument => Error::InvalidArgument, - policy::error::Error::InvalidServiceType(s) => Error::InvalidServiceType(s), - policy::error::Error::IAMActionNotAllowed => Error::IAMActionNotAllowed, - policy::error::Error::InvalidExpiration => Error::InvalidExpiration, - policy::error::Error::NoAccessKey => Error::NoAccessKey, - policy::error::Error::InvalidToken => Error::InvalidToken, - policy::error::Error::InvalidAccessKey => Error::InvalidAccessKey, - policy::error::Error::NoSecretKeyWithAccessKey => Error::NoSecretKeyWithAccessKey, - policy::error::Error::NoAccessKeyWithSecretKey => Error::NoAccessKeyWithSecretKey, - policy::error::Error::Io(e) => Error::Io(e), - policy::error::Error::JWTError(e) => Error::JWTError(e), - policy::error::Error::NoSuchUser(s) => Error::NoSuchUser(s), - policy::error::Error::NoSuchAccount(s) => Error::NoSuchAccount(s), - policy::error::Error::NoSuchServiceAccount(s) => Error::NoSuchServiceAccount(s), - policy::error::Error::NoSuchTempAccount(s) => Error::NoSuchTempAccount(s), - policy::error::Error::NoSuchGroup(s) => Error::NoSuchGroup(s), - policy::error::Error::NoSuchPolicy => Error::NoSuchPolicy, - policy::error::Error::PolicyInUse => Error::PolicyInUse, - policy::error::Error::GroupNotEmpty => Error::GroupNotEmpty, - policy::error::Error::InvalidAccessKeyLength => Error::InvalidAccessKeyLength, - policy::error::Error::InvalidSecretKeyLength => Error::InvalidSecretKeyLength, - policy::error::Error::ContainsReservedChars => Error::ContainsReservedChars, - policy::error::Error::GroupNameContainsReservedChars => Error::GroupNameContainsReservedChars, - policy::error::Error::CredNotInitialized => Error::CredNotInitialized, - policy::error::Error::IamSysNotInitialized => Error::IamSysNotInitialized, - policy::error::Error::PolicyError(e) => Error::PolicyError(e), - policy::error::Error::StringError(s) => Error::StringError(s), - policy::error::Error::CryptoError(e) => Error::CryptoError(e), - policy::error::Error::ErrCredMalformed => Error::ErrCredMalformed, + rustfs_policy::error::Error::PolicyTooLarge => Error::PolicyTooLarge, + rustfs_policy::error::Error::InvalidArgument => Error::InvalidArgument, + rustfs_policy::error::Error::InvalidServiceType(s) => Error::InvalidServiceType(s), + rustfs_policy::error::Error::IAMActionNotAllowed => Error::IAMActionNotAllowed, + rustfs_policy::error::Error::InvalidExpiration => Error::InvalidExpiration, + rustfs_policy::error::Error::NoAccessKey => Error::NoAccessKey, + rustfs_policy::error::Error::InvalidToken => Error::InvalidToken, + rustfs_policy::error::Error::InvalidAccessKey => Error::InvalidAccessKey, + rustfs_policy::error::Error::NoSecretKeyWithAccessKey => Error::NoSecretKeyWithAccessKey, + rustfs_policy::error::Error::NoAccessKeyWithSecretKey => Error::NoAccessKeyWithSecretKey, + rustfs_policy::error::Error::Io(e) => Error::Io(e), + rustfs_policy::error::Error::JWTError(e) => Error::JWTError(e), + rustfs_policy::error::Error::NoSuchUser(s) => Error::NoSuchUser(s), + rustfs_policy::error::Error::NoSuchAccount(s) => Error::NoSuchAccount(s), + rustfs_policy::error::Error::NoSuchServiceAccount(s) => Error::NoSuchServiceAccount(s), + rustfs_policy::error::Error::NoSuchTempAccount(s) => Error::NoSuchTempAccount(s), + rustfs_policy::error::Error::NoSuchGroup(s) => Error::NoSuchGroup(s), + rustfs_policy::error::Error::NoSuchPolicy => Error::NoSuchPolicy, + rustfs_policy::error::Error::PolicyInUse => Error::PolicyInUse, + rustfs_policy::error::Error::GroupNotEmpty => Error::GroupNotEmpty, + rustfs_policy::error::Error::InvalidAccessKeyLength => Error::InvalidAccessKeyLength, + rustfs_policy::error::Error::InvalidSecretKeyLength => Error::InvalidSecretKeyLength, + rustfs_policy::error::Error::ContainsReservedChars => Error::ContainsReservedChars, + rustfs_policy::error::Error::GroupNameContainsReservedChars => Error::GroupNameContainsReservedChars, + rustfs_policy::error::Error::CredNotInitialized => Error::CredNotInitialized, + rustfs_policy::error::Error::IamSysNotInitialized => Error::IamSysNotInitialized, + rustfs_policy::error::Error::PolicyError(e) => Error::PolicyError(e), + rustfs_policy::error::Error::StringError(s) => Error::StringError(s), + rustfs_policy::error::Error::CryptoError(e) => Error::CryptoError(e), + rustfs_policy::error::Error::ErrCredMalformed => Error::ErrCredMalformed, } } } @@ -325,18 +325,18 @@ mod tests { #[test] fn test_iam_error_from_storage_error() { // Test conversion from StorageError - let storage_error = ecstore::error::StorageError::ConfigNotFound; + let storage_error = rustfs_ecstore::error::StorageError::ConfigNotFound; let iam_error: Error = storage_error.into(); assert_eq!(iam_error, Error::ConfigNotFound); // Test reverse conversion - let back_to_storage: ecstore::error::StorageError = iam_error.into(); - assert_eq!(back_to_storage, ecstore::error::StorageError::ConfigNotFound); + let back_to_storage: rustfs_ecstore::error::StorageError = iam_error.into(); + assert_eq!(back_to_storage, rustfs_ecstore::error::StorageError::ConfigNotFound); } #[test] fn test_iam_error_from_policy_error() { - use policy::error::Error as PolicyError; + use rustfs_policy::error::Error as PolicyError; let policy_errors = vec![ (PolicyError::NoSuchUser("user1".to_string()), Error::NoSuchUser("user1".to_string())), diff --git a/iam/src/lib.rs b/crates/iam/src/lib.rs similarity index 97% rename from iam/src/lib.rs rename to crates/iam/src/lib.rs index b1d9e722..c3ca5656 100644 --- a/iam/src/lib.rs +++ b/crates/iam/src/lib.rs @@ -13,8 +13,8 @@ // limitations under the License. use crate::error::{Error, Result}; -use ecstore::store::ECStore; use manager::IamCache; +use rustfs_ecstore::store::ECStore; use std::sync::{Arc, OnceLock}; use store::object::ObjectStore; use sys::IamSys; diff --git a/iam/src/manager.rs b/crates/iam/src/manager.rs similarity index 98% rename from iam/src/manager.rs rename to crates/iam/src/manager.rs index 6535367a..c0831ce1 100644 --- a/iam/src/manager.rs +++ b/crates/iam/src/manager.rs @@ -22,9 +22,9 @@ use crate::{ UpdateServiceAccountOpts, }, }; -use ecstore::global::get_global_action_cred; -use madmin::{AccountStatus, AddOrUpdateUserReq, GroupDesc}; -use policy::{ +use rustfs_ecstore::global::get_global_action_cred; +use rustfs_madmin::{AccountStatus, AddOrUpdateUserReq, GroupDesc}; +use rustfs_policy::{ arn::ARN, auth::{self, Credentials, UserIdentity, get_claims_from_token_with_secret, is_secret_key_valid, jwt_sign}, format::Format, @@ -543,7 +543,7 @@ where if m.contains_key(SESSION_POLICY_NAME) && nosp { m.remove(SESSION_POLICY_NAME); - m.insert(iam_policy_claim_name_sa(), serde_json::Value::String(INHERITED_POLICY_TYPE.to_owned())); + m.insert(iam_policy_claim_name_sa(), Value::String(INHERITED_POLICY_TYPE.to_owned())); } if let Some(session_policy) = &opts.session_policy { @@ -554,12 +554,12 @@ where return Err(Error::PolicyTooLarge); } - m.insert(SESSION_POLICY_NAME.to_owned(), serde_json::Value::String(base64_encode(&policy_buf))); - m.insert(iam_policy_claim_name_sa(), serde_json::Value::String(EMBEDDED_POLICY_TYPE.to_owned())); + m.insert(SESSION_POLICY_NAME.to_owned(), Value::String(base64_encode(&policy_buf))); + m.insert(iam_policy_claim_name_sa(), Value::String(EMBEDDED_POLICY_TYPE.to_owned())); } } - m.insert("accessKey".to_owned(), serde_json::Value::String(name.to_owned())); + m.insert("accessKey".to_owned(), Value::String(name.to_owned())); cr.session_token = jwt_sign(&m, &cr.secret_key)?; @@ -833,7 +833,7 @@ where Ok(OffsetDateTime::now_utc()) } - pub async fn get_user_info(&self, name: &str) -> Result { + pub async fn get_user_info(&self, name: &str) -> Result { let users = self.cache.users.load(); let policies = self.cache.user_policies.load(); let group_members = self.cache.user_group_memberships.load(); @@ -847,11 +847,11 @@ where return Err(Error::IAMActionNotAllowed); } - let mut uinfo = madmin::UserInfo { + let mut uinfo = rustfs_madmin::UserInfo { status: if u.credentials.is_valid() { - madmin::AccountStatus::Enabled + AccountStatus::Enabled } else { - madmin::AccountStatus::Disabled + AccountStatus::Disabled }, updated_at: u.update_at, ..Default::default() @@ -870,7 +870,7 @@ where } // returns all users (not STS or service accounts) - pub async fn get_users(&self) -> Result> { + pub async fn get_users(&self) -> Result> { let mut m = HashMap::new(); let users = self.cache.users.load(); @@ -882,11 +882,11 @@ where continue; } - let mut u = madmin::UserInfo { + let mut u = rustfs_madmin::UserInfo { status: if v.credentials.is_valid() { - madmin::AccountStatus::Enabled + AccountStatus::Enabled } else { - madmin::AccountStatus::Disabled + AccountStatus::Disabled }, updated_at: v.update_at, ..Default::default() @@ -906,7 +906,7 @@ where Ok(m) } - pub async fn get_bucket_users(&self, bucket_name: &str) -> Result> { + pub async fn get_bucket_users(&self, bucket_name: &str) -> Result> { let users = self.cache.users.load(); let policies_cache = self.cache.user_policies.load(); let group_members = self.cache.user_group_memberships.load(); @@ -937,12 +937,12 @@ where continue; } - let mut u = madmin::UserInfo { + let mut u = rustfs_madmin::UserInfo { policy_name: Some(matched_policies), status: if v.credentials.is_valid() { - madmin::AccountStatus::Enabled + AccountStatus::Enabled } else { - madmin::AccountStatus::Disabled + AccountStatus::Disabled }, updated_at: v.update_at, ..Default::default() @@ -1642,7 +1642,7 @@ fn filter_policies(cache: &Cache, policy_name: &str, bucket_name: &str) -> (Stri #[cfg(test)] mod tests { use super::*; - use policy::policy::{Policy, PolicyDoc}; + use rustfs_policy::policy::{Policy, PolicyDoc}; use serde_json::json; use std::collections::HashMap; diff --git a/iam/src/store.rs b/crates/iam/src/store.rs similarity index 98% rename from iam/src/store.rs rename to crates/iam/src/store.rs index d8ddb0e5..76d5306d 100644 --- a/iam/src/store.rs +++ b/crates/iam/src/store.rs @@ -16,7 +16,7 @@ pub mod object; use crate::cache::Cache; use crate::error::Result; -use policy::{auth::UserIdentity, policy::PolicyDoc}; +use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc}; use serde::{Deserialize, Serialize, de::DeserializeOwned}; use std::collections::{HashMap, HashSet}; use time::OffsetDateTime; diff --git a/iam/src/store/object.rs b/crates/iam/src/store/object.rs similarity index 99% rename from iam/src/store/object.rs rename to crates/iam/src/store/object.rs index 9802fc79..1ca416b5 100644 --- a/iam/src/store/object.rs +++ b/crates/iam/src/store/object.rs @@ -19,7 +19,9 @@ use crate::{ error::{is_err_no_such_policy, is_err_no_such_user}, manager::{extract_jwt_claims, get_default_policyes}, }; -use ecstore::{ +use futures::future::join_all; +use lazy_static::lazy_static; +use rustfs_ecstore::{ config::{ RUSTFS_CONFIG_PREFIX, com::{delete_config, read_config, read_config_with_metadata, save_config}, @@ -29,9 +31,7 @@ use ecstore::{ store_api::{ObjectInfo, ObjectOptions}, store_list_objects::{ObjectInfoOrErr, WalkOptions}, }; -use futures::future::join_all; -use lazy_static::lazy_static; -use policy::{auth::UserIdentity, policy::PolicyDoc}; +use rustfs_policy::{auth::UserIdentity, policy::PolicyDoc}; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use serde::{Serialize, de::DeserializeOwned}; use std::{collections::HashMap, sync::Arc}; @@ -127,12 +127,12 @@ impl ObjectStore { } fn decrypt_data(data: &[u8]) -> Result> { - let de = crypto::decrypt_data(get_global_action_cred().unwrap_or_default().secret_key.as_bytes(), data)?; + let de = rustfs_crypto::decrypt_data(get_global_action_cred().unwrap_or_default().secret_key.as_bytes(), data)?; Ok(de) } fn encrypt_data(data: &[u8]) -> Result> { - let en = crypto::encrypt_data(get_global_action_cred().unwrap_or_default().secret_key.as_bytes(), data)?; + let en = rustfs_crypto::encrypt_data(get_global_action_cred().unwrap_or_default().secret_key.as_bytes(), data)?; Ok(en) } diff --git a/iam/src/sys.rs b/crates/iam/src/sys.rs similarity index 95% rename from iam/src/sys.rs rename to crates/iam/src/sys.rs index d5a5e375..0b6947c5 100644 --- a/iam/src/sys.rs +++ b/crates/iam/src/sys.rs @@ -23,24 +23,17 @@ use crate::store::GroupInfo; use crate::store::MappedPolicy; use crate::store::Store; use crate::store::UserType; -use ecstore::global::get_global_action_cred; -use madmin::AddOrUpdateUserReq; -use madmin::GroupDesc; -use policy::arn::ARN; -use policy::auth::ACCOUNT_ON; -use policy::auth::Credentials; -use policy::auth::UserIdentity; -use policy::auth::contains_reserved_chars; -use policy::auth::create_new_credentials_with_metadata; -use policy::auth::generate_credentials; -use policy::auth::is_access_key_valid; -use policy::auth::is_secret_key_valid; -use policy::policy::Args; -use policy::policy::EMBEDDED_POLICY_TYPE; -use policy::policy::INHERITED_POLICY_TYPE; -use policy::policy::Policy; -use policy::policy::PolicyDoc; -use policy::policy::iam_policy_claim_name_sa; +use rustfs_ecstore::global::get_global_action_cred; +use rustfs_madmin::AddOrUpdateUserReq; +use rustfs_madmin::GroupDesc; +use rustfs_policy::arn::ARN; +use rustfs_policy::auth::Credentials; +use rustfs_policy::auth::{ + ACCOUNT_ON, UserIdentity, contains_reserved_chars, create_new_credentials_with_metadata, generate_credentials, + is_access_key_valid, is_secret_key_valid, +}; +use rustfs_policy::policy::Args; +use rustfs_policy::policy::{EMBEDDED_POLICY_TYPE, INHERITED_POLICY_TYPE, Policy, PolicyDoc, iam_policy_claim_name_sa}; use rustfs_utils::crypto::{base64_decode, base64_encode}; use serde_json::Value; use serde_json::json; @@ -117,12 +110,12 @@ impl IamSys { Ok(()) } - pub async fn info_policy(&self, name: &str) -> Result { + pub async fn info_policy(&self, name: &str) -> Result { let d = self.store.get_policy_doc(name).await?; let pdata = serde_json::to_string(&d.policy)?; - Ok(madmin::PolicyInfo { + Ok(rustfs_madmin::PolicyInfo { policy_name: name.to_string(), policy: json!(pdata), create_date: d.create_date, @@ -174,11 +167,11 @@ impl IamSys { self.store.merge_policies(name).await.0 } - pub async fn list_bucket_users(&self, bucket_name: &str) -> Result> { + pub async fn list_bucket_users(&self, bucket_name: &str) -> Result> { self.store.get_bucket_users(bucket_name).await } - pub async fn list_users(&self) -> Result> { + pub async fn list_users(&self) -> Result> { self.store.get_users().await } @@ -209,11 +202,11 @@ impl IamSys { } } - pub async fn get_user_info(&self, name: &str) -> Result { + pub async fn get_user_info(&self, name: &str) -> Result { self.store.get_user_info(name).await } - pub async fn set_user_status(&self, name: &str, status: madmin::AccountStatus) -> Result { + pub async fn set_user_status(&self, name: &str, status: rustfs_madmin::AccountStatus) -> Result { self.store.set_user_status(name, status).await // TODO: notification } diff --git a/iam/src/utils.rs b/crates/iam/src/utils.rs similarity index 100% rename from iam/src/utils.rs rename to crates/iam/src/utils.rs diff --git a/common/lock/Cargo.toml b/crates/lock/Cargo.toml similarity index 85% rename from common/lock/Cargo.toml rename to crates/lock/Cargo.toml index 8e6a7c5d..1b934c97 100644 --- a/common/lock/Cargo.toml +++ b/crates/lock/Cargo.toml @@ -13,25 +13,25 @@ # limitations under the License. [package] -name = "lock" +name = "rustfs-lock" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true [lints] workspace = true [dependencies] async-trait.workspace = true -backon.workspace = true -common.workspace = true lazy_static.workspace = true -protos.workspace = true +rustfs-protos.workspace = true rand.workspace = true serde.workspace = true serde_json.workspace = true tokio.workspace = true tonic.workspace = true tracing.workspace = true -tracing-error.workspace = true url.workspace = true uuid.workspace = true \ No newline at end of file diff --git a/common/lock/src/drwmutex.rs b/crates/lock/src/drwmutex.rs similarity index 99% rename from common/lock/src/drwmutex.rs rename to crates/lock/src/drwmutex.rs index deb7b91d..bce263c3 100644 --- a/common/lock/src/drwmutex.rs +++ b/crates/lock/src/drwmutex.rs @@ -13,7 +13,6 @@ // limitations under the License. use std::time::{Duration, Instant}; - use tokio::{sync::mpsc::Sender, time::sleep}; use tracing::{info, warn}; diff --git a/common/lock/src/lib.rs b/crates/lock/src/lib.rs similarity index 99% rename from common/lock/src/lib.rs rename to crates/lock/src/lib.rs index 396f3b00..e9e9b956 100644 --- a/common/lock/src/lib.rs +++ b/crates/lock/src/lib.rs @@ -13,14 +13,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::sync::Arc; - use async_trait::async_trait; use lazy_static::lazy_static; use local_locker::LocalLocker; use lock_args::LockArgs; use remote_client::RemoteClient; use std::io::Result; +use std::sync::Arc; use tokio::sync::RwLock; pub mod drwmutex; diff --git a/common/lock/src/local_locker.rs b/crates/lock/src/local_locker.rs similarity index 100% rename from common/lock/src/local_locker.rs rename to crates/lock/src/local_locker.rs diff --git a/common/lock/src/lock_args.rs b/crates/lock/src/lock_args.rs similarity index 99% rename from common/lock/src/lock_args.rs rename to crates/lock/src/lock_args.rs index 86850962..c8ee597b 100644 --- a/common/lock/src/lock_args.rs +++ b/crates/lock/src/lock_args.rs @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::fmt::Display; - use serde::{Deserialize, Serialize}; +use std::fmt::Display; #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct LockArgs { diff --git a/common/lock/src/lrwmutex.rs b/crates/lock/src/lrwmutex.rs similarity index 99% rename from common/lock/src/lrwmutex.rs rename to crates/lock/src/lrwmutex.rs index 5f0779b1..d701ef12 100644 --- a/common/lock/src/lrwmutex.rs +++ b/crates/lock/src/lrwmutex.rs @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::time::{Duration, Instant}; - use rand::Rng; +use std::time::{Duration, Instant}; use tokio::{sync::RwLock, time::sleep}; use tracing::info; diff --git a/common/lock/src/namespace_lock.rs b/crates/lock/src/namespace_lock.rs similarity index 99% rename from common/lock/src/namespace_lock.rs rename to crates/lock/src/namespace_lock.rs index c2fe424e..f4c06a21 100644 --- a/common/lock/src/namespace_lock.rs +++ b/crates/lock/src/namespace_lock.rs @@ -12,9 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use std::{collections::HashMap, path::Path, sync::Arc, time::Duration}; - use async_trait::async_trait; +use std::{collections::HashMap, path::Path, sync::Arc, time::Duration}; use tokio::sync::RwLock; use uuid::Uuid; diff --git a/common/lock/src/remote_client.rs b/crates/lock/src/remote_client.rs similarity index 98% rename from common/lock/src/remote_client.rs rename to crates/lock/src/remote_client.rs index 10fd0a5a..789bf407 100644 --- a/common/lock/src/remote_client.rs +++ b/crates/lock/src/remote_client.rs @@ -12,14 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. +use crate::{Locker, lock_args::LockArgs}; use async_trait::async_trait; -use protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest}; +use rustfs_protos::{node_service_time_out_client, proto_gen::node_service::GenerallyLockRequest}; use std::io::{Error, Result}; use tonic::Request; use tracing::info; -use crate::{Locker, lock_args::LockArgs}; - #[derive(Debug, Clone)] pub struct RemoteClient { addr: String, diff --git a/madmin/Cargo.toml b/crates/madmin/Cargo.toml similarity index 91% rename from madmin/Cargo.toml rename to crates/madmin/Cargo.toml index be7779f1..36d4c221 100644 --- a/madmin/Cargo.toml +++ b/crates/madmin/Cargo.toml @@ -13,7 +13,7 @@ # limitations under the License. [package] -name = "madmin" +name = "rustfs-madmin" edition.workspace = true license.workspace = true repository.workspace = true @@ -25,11 +25,8 @@ workspace = true [dependencies] chrono.workspace = true -common.workspace = true humantime.workspace = true hyper.workspace = true serde.workspace = true serde_json.workspace = true time.workspace = true -tracing.workspace = true -s3s.workspace = true diff --git a/madmin/src/group.rs b/crates/madmin/src/group.rs similarity index 100% rename from madmin/src/group.rs rename to crates/madmin/src/group.rs diff --git a/madmin/src/heal_commands.rs b/crates/madmin/src/heal_commands.rs similarity index 100% rename from madmin/src/heal_commands.rs rename to crates/madmin/src/heal_commands.rs diff --git a/madmin/src/health.rs b/crates/madmin/src/health.rs similarity index 100% rename from madmin/src/health.rs rename to crates/madmin/src/health.rs diff --git a/madmin/src/info_commands.rs b/crates/madmin/src/info_commands.rs similarity index 100% rename from madmin/src/info_commands.rs rename to crates/madmin/src/info_commands.rs diff --git a/madmin/src/lib.rs b/crates/madmin/src/lib.rs similarity index 100% rename from madmin/src/lib.rs rename to crates/madmin/src/lib.rs diff --git a/madmin/src/metrics.rs b/crates/madmin/src/metrics.rs similarity index 100% rename from madmin/src/metrics.rs rename to crates/madmin/src/metrics.rs diff --git a/madmin/src/net/mod.rs b/crates/madmin/src/net/mod.rs similarity index 100% rename from madmin/src/net/mod.rs rename to crates/madmin/src/net/mod.rs diff --git a/madmin/src/policy.rs b/crates/madmin/src/policy.rs similarity index 100% rename from madmin/src/policy.rs rename to crates/madmin/src/policy.rs diff --git a/madmin/src/service_commands.rs b/crates/madmin/src/service_commands.rs similarity index 100% rename from madmin/src/service_commands.rs rename to crates/madmin/src/service_commands.rs diff --git a/madmin/src/trace.rs b/crates/madmin/src/trace.rs similarity index 100% rename from madmin/src/trace.rs rename to crates/madmin/src/trace.rs diff --git a/madmin/src/user.rs b/crates/madmin/src/user.rs similarity index 100% rename from madmin/src/user.rs rename to crates/madmin/src/user.rs diff --git a/madmin/src/utils.rs b/crates/madmin/src/utils.rs similarity index 100% rename from madmin/src/utils.rs rename to crates/madmin/src/utils.rs diff --git a/crates/notify/Cargo.toml b/crates/notify/Cargo.toml index e7fc1266..6c427835 100644 --- a/crates/notify/Cargo.toml +++ b/crates/notify/Cargo.toml @@ -26,7 +26,7 @@ rustfs-utils = { workspace = true, features = ["path", "sys"] } async-trait = { workspace = true } chrono = { workspace = true, features = ["serde"] } dashmap = { workspace = true } -ecstore = { workspace = true } +rustfs-ecstore = { workspace = true } form_urlencoded = { workspace = true } once_cell = { workspace = true } quick-xml = { workspace = true, features = ["serialize", "async-tokio"] } @@ -48,7 +48,7 @@ wildmatch = { workspace = true, features = ["serde"] } [dev-dependencies] tokio = { workspace = true, features = ["test-util"] } -reqwest = { workspace = true, default-features = false, features = ["rustls-tls", "charset", "http2", "system-proxy", "stream", "json", "blocking"] } +reqwest = { workspace = true } axum = { workspace = true } [lints] diff --git a/crates/notify/examples/full_demo.rs b/crates/notify/examples/full_demo.rs index 872c5f02..da51b0b8 100644 --- a/crates/notify/examples/full_demo.rs +++ b/crates/notify/examples/full_demo.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::config::{Config, ENABLE_KEY, ENABLE_ON, KV, KVS}; use rustfs_config::notify::{ DEFAULT_LIMIT, DEFAULT_TARGET, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, WEBHOOK_AUTH_TOKEN, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, }; +use rustfs_ecstore::config::{Config, ENABLE_KEY, ENABLE_ON, KV, KVS}; use rustfs_notify::arn::TargetID; use rustfs_notify::{BucketNotificationConfig, Event, EventName, LogLevel, NotificationError, init_logger}; use rustfs_notify::{initialize, notification_system}; diff --git a/crates/notify/examples/full_demo_one.rs b/crates/notify/examples/full_demo_one.rs index e3e756e8..80b80c02 100644 --- a/crates/notify/examples/full_demo_one.rs +++ b/crates/notify/examples/full_demo_one.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::config::{Config, ENABLE_KEY, ENABLE_ON, KV, KVS}; // Using Global Accessories use rustfs_config::notify::{ DEFAULT_LIMIT, DEFAULT_TARGET, MQTT_BROKER, MQTT_PASSWORD, MQTT_QOS, MQTT_QUEUE_DIR, MQTT_QUEUE_LIMIT, MQTT_TOPIC, MQTT_USERNAME, NOTIFY_MQTT_SUB_SYS, NOTIFY_WEBHOOK_SUB_SYS, WEBHOOK_AUTH_TOKEN, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, }; +use rustfs_ecstore::config::{Config, ENABLE_KEY, ENABLE_ON, KV, KVS}; use rustfs_notify::arn::TargetID; use rustfs_notify::{BucketNotificationConfig, Event, EventName, LogLevel, NotificationError, init_logger}; use rustfs_notify::{initialize, notification_system}; diff --git a/crates/notify/src/event.rs b/crates/notify/src/event.rs index 21d42510..308de9d9 100644 --- a/crates/notify/src/event.rs +++ b/crates/notify/src/event.rs @@ -547,7 +547,7 @@ pub struct EventLog { pub struct EventArgs { pub event_name: EventName, pub bucket_name: String, - pub object: ecstore::store_api::ObjectInfo, + pub object: rustfs_ecstore::store_api::ObjectInfo, pub req_params: HashMap, pub resp_elements: HashMap, pub version_id: String, diff --git a/crates/notify/src/factory.rs b/crates/notify/src/factory.rs index 01fd496e..a33ba396 100644 --- a/crates/notify/src/factory.rs +++ b/crates/notify/src/factory.rs @@ -17,7 +17,6 @@ use crate::{ target::{Target, mqtt::MQTTArgs, webhook::WebhookArgs}, }; use async_trait::async_trait; -use ecstore::config::{ENABLE_KEY, ENABLE_ON, KVS}; use rumqttc::QoS; use rustfs_config::notify::{ DEFAULT_DIR, DEFAULT_LIMIT, ENV_MQTT_BROKER, ENV_MQTT_ENABLE, ENV_MQTT_KEEP_ALIVE_INTERVAL, ENV_MQTT_PASSWORD, ENV_MQTT_QOS, @@ -28,6 +27,7 @@ use rustfs_config::notify::{ WEBHOOK_CLIENT_CERT, WEBHOOK_CLIENT_KEY, WEBHOOK_ENDPOINT, WEBHOOK_QUEUE_DIR, WEBHOOK_QUEUE_LIMIT, }; use rustfs_config::{DEFAULT_DELIMITER, ENV_WORD_DELIMITER_DASH}; +use rustfs_ecstore::config::{ENABLE_KEY, ENABLE_ON, KVS}; use std::time::Duration; use tracing::{debug, warn}; use url::Url; diff --git a/crates/notify/src/global.rs b/crates/notify/src/global.rs index 68b1f23e..b55dd1f3 100644 --- a/crates/notify/src/global.rs +++ b/crates/notify/src/global.rs @@ -13,8 +13,8 @@ // limitations under the License. use crate::{Event, EventArgs, NotificationError, NotificationSystem}; -use ecstore::config::Config; use once_cell::sync::Lazy; +use rustfs_ecstore::config::Config; use std::sync::{Arc, OnceLock}; use tracing::instrument; diff --git a/crates/notify/src/integration.rs b/crates/notify/src/integration.rs index 3c78597f..4ef18a08 100644 --- a/crates/notify/src/integration.rs +++ b/crates/notify/src/integration.rs @@ -18,7 +18,7 @@ use crate::{ Event, EventName, StoreError, Target, error::NotificationError, notifier::EventNotifier, registry::TargetRegistry, rules::BucketNotificationConfig, stream, }; -use ecstore::config::{Config, KVS}; +use rustfs_ecstore::config::{Config, KVS}; use std::collections::HashMap; use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -196,11 +196,11 @@ impl NotificationSystem { where F: FnMut(&mut Config) -> bool, // The closure returns a boolean value indicating whether the configuration has been changed { - let Some(store) = ecstore::global::new_object_layer_fn() else { + let Some(store) = rustfs_ecstore::global::new_object_layer_fn() else { return Err(NotificationError::ServerNotInitialized); }; - let mut new_config = ecstore::config::com::read_config_without_migrate(store.clone()) + let mut new_config = rustfs_ecstore::config::com::read_config_without_migrate(store.clone()) .await .map_err(|e| NotificationError::ReadConfig(e.to_string()))?; @@ -210,7 +210,7 @@ impl NotificationSystem { return Ok(()); } - if let Err(e) = ecstore::config::com::save_server_config(store, &new_config).await { + if let Err(e) = rustfs_ecstore::config::com::save_server_config(store, &new_config).await { error!("Failed to save config: {}", e); return Err(NotificationError::SaveConfig(e.to_string())); } diff --git a/crates/notify/src/registry.rs b/crates/notify/src/registry.rs index 77df0963..828173c3 100644 --- a/crates/notify/src/registry.rs +++ b/crates/notify/src/registry.rs @@ -18,9 +18,9 @@ use crate::{ factory::{MQTTTargetFactory, TargetFactory, WebhookTargetFactory}, target::Target, }; -use ecstore::config::{Config, ENABLE_KEY, ENABLE_OFF, ENABLE_ON, KVS}; use rustfs_config::notify::NOTIFY_ROUTE_PREFIX; use rustfs_config::{DEFAULT_DELIMITER, ENV_PREFIX}; +use rustfs_ecstore::config::{Config, ENABLE_KEY, ENABLE_OFF, ENABLE_ON, KVS}; use std::collections::HashMap; use tracing::{debug, error, info}; diff --git a/policy/Cargo.toml b/crates/policy/Cargo.toml similarity index 82% rename from policy/Cargo.toml rename to crates/policy/Cargo.toml index 59ed9c7c..dd464026 100644 --- a/policy/Cargo.toml +++ b/crates/policy/Cargo.toml @@ -13,7 +13,7 @@ # limitations under the License. [package] -name = "policy" +name = "rustfs-policy" edition.workspace = true license.workspace = true repository.workspace = true @@ -28,22 +28,14 @@ tokio.workspace = true time = { workspace = true, features = ["serde-human-readable"] } serde = { workspace = true, features = ["derive", "rc"] } serde_json.workspace = true -async-trait.workspace = true thiserror.workspace = true strum = { workspace = true, features = ["derive"] } -arc-swap = "1.7.1" -crypto = { path = "../crypto" } +rustfs-crypto = { workspace = true } ipnetwork = { workspace = true, features = ["serde"] } -itertools = { workspace = true } -futures.workspace = true rand.workspace = true base64-simd = { workspace = true } jsonwebtoken = { workspace = true } -tracing.workspace = true -madmin.workspace = true -lazy_static.workspace = true regex = { workspace = true } -common.workspace = true [dev-dependencies] test-case.workspace = true diff --git a/policy/src/arn.rs b/crates/policy/src/arn.rs similarity index 100% rename from policy/src/arn.rs rename to crates/policy/src/arn.rs diff --git a/policy/src/auth.rs b/crates/policy/src/auth.rs similarity index 100% rename from policy/src/auth.rs rename to crates/policy/src/auth.rs diff --git a/policy/src/auth/credentials.rs b/crates/policy/src/auth/credentials.rs similarity index 99% rename from policy/src/auth/credentials.rs rename to crates/policy/src/auth/credentials.rs index 64c42c08..6a1d3d8b 100644 --- a/policy/src/auth/credentials.rs +++ b/crates/policy/src/auth/credentials.rs @@ -404,7 +404,7 @@ impl TryFrom for Credentials { }; if !value.secret_key.is_empty() { - let session_token = crypto::jwt_encode(value.access_key.as_bytes(), &claim) + let session_token = rustfs_crypto::jwt_encode(value.access_key.as_bytes(), &claim) .map_err(|_| Error::other("session policy is too large"))?; cred.session_token = session_token; // cred.expiration = Some( diff --git a/policy/src/error.rs b/crates/policy/src/error.rs similarity index 99% rename from policy/src/error.rs rename to crates/policy/src/error.rs index 90d5fd03..e9b39920 100644 --- a/policy/src/error.rs +++ b/crates/policy/src/error.rs @@ -25,7 +25,7 @@ pub enum Error { StringError(String), #[error("crypto: {0}")] - CryptoError(#[from] crypto::Error), + CryptoError(#[from] rustfs_crypto::Error), #[error("user '{0}' does not exist")] NoSuchUser(String), @@ -211,7 +211,7 @@ mod tests { #[test] fn test_policy_error_from_crypto_error() { // Test conversion from crypto::Error - use an actual variant - let crypto_error = crypto::Error::ErrUnexpectedHeader; + let crypto_error = rustfs_crypto::Error::ErrUnexpectedHeader; let policy_error: Error = crypto_error.into(); match policy_error { diff --git a/policy/src/format.rs b/crates/policy/src/format.rs similarity index 100% rename from policy/src/format.rs rename to crates/policy/src/format.rs diff --git a/policy/src/lib.rs b/crates/policy/src/lib.rs similarity index 100% rename from policy/src/lib.rs rename to crates/policy/src/lib.rs diff --git a/policy/src/policy.rs b/crates/policy/src/policy.rs similarity index 100% rename from policy/src/policy.rs rename to crates/policy/src/policy.rs diff --git a/policy/src/policy/action.rs b/crates/policy/src/policy/action.rs similarity index 100% rename from policy/src/policy/action.rs rename to crates/policy/src/policy/action.rs diff --git a/policy/src/policy/doc.rs b/crates/policy/src/policy/doc.rs similarity index 100% rename from policy/src/policy/doc.rs rename to crates/policy/src/policy/doc.rs diff --git a/policy/src/policy/effect.rs b/crates/policy/src/policy/effect.rs similarity index 100% rename from policy/src/policy/effect.rs rename to crates/policy/src/policy/effect.rs diff --git a/policy/src/policy/function.rs b/crates/policy/src/policy/function.rs similarity index 100% rename from policy/src/policy/function.rs rename to crates/policy/src/policy/function.rs diff --git a/policy/src/policy/function/addr.rs b/crates/policy/src/policy/function/addr.rs similarity index 100% rename from policy/src/policy/function/addr.rs rename to crates/policy/src/policy/function/addr.rs diff --git a/policy/src/policy/function/binary.rs b/crates/policy/src/policy/function/binary.rs similarity index 100% rename from policy/src/policy/function/binary.rs rename to crates/policy/src/policy/function/binary.rs diff --git a/policy/src/policy/function/bool_null.rs b/crates/policy/src/policy/function/bool_null.rs similarity index 100% rename from policy/src/policy/function/bool_null.rs rename to crates/policy/src/policy/function/bool_null.rs diff --git a/policy/src/policy/function/condition.rs b/crates/policy/src/policy/function/condition.rs similarity index 100% rename from policy/src/policy/function/condition.rs rename to crates/policy/src/policy/function/condition.rs diff --git a/policy/src/policy/function/date.rs b/crates/policy/src/policy/function/date.rs similarity index 100% rename from policy/src/policy/function/date.rs rename to crates/policy/src/policy/function/date.rs diff --git a/policy/src/policy/function/func.rs b/crates/policy/src/policy/function/func.rs similarity index 100% rename from policy/src/policy/function/func.rs rename to crates/policy/src/policy/function/func.rs diff --git a/policy/src/policy/function/key.rs b/crates/policy/src/policy/function/key.rs similarity index 100% rename from policy/src/policy/function/key.rs rename to crates/policy/src/policy/function/key.rs diff --git a/policy/src/policy/function/key_name.rs b/crates/policy/src/policy/function/key_name.rs similarity index 100% rename from policy/src/policy/function/key_name.rs rename to crates/policy/src/policy/function/key_name.rs diff --git a/policy/src/policy/function/number.rs b/crates/policy/src/policy/function/number.rs similarity index 100% rename from policy/src/policy/function/number.rs rename to crates/policy/src/policy/function/number.rs diff --git a/policy/src/policy/function/string.rs b/crates/policy/src/policy/function/string.rs similarity index 100% rename from policy/src/policy/function/string.rs rename to crates/policy/src/policy/function/string.rs diff --git a/policy/src/policy/id.rs b/crates/policy/src/policy/id.rs similarity index 100% rename from policy/src/policy/id.rs rename to crates/policy/src/policy/id.rs diff --git a/policy/src/policy/policy.rs b/crates/policy/src/policy/policy.rs similarity index 100% rename from policy/src/policy/policy.rs rename to crates/policy/src/policy/policy.rs diff --git a/policy/src/policy/principal.rs b/crates/policy/src/policy/principal.rs similarity index 100% rename from policy/src/policy/principal.rs rename to crates/policy/src/policy/principal.rs diff --git a/policy/src/policy/resource.rs b/crates/policy/src/policy/resource.rs similarity index 100% rename from policy/src/policy/resource.rs rename to crates/policy/src/policy/resource.rs diff --git a/policy/src/policy/statement.rs b/crates/policy/src/policy/statement.rs similarity index 100% rename from policy/src/policy/statement.rs rename to crates/policy/src/policy/statement.rs diff --git a/policy/src/policy/utils.rs b/crates/policy/src/policy/utils.rs similarity index 100% rename from policy/src/policy/utils.rs rename to crates/policy/src/policy/utils.rs diff --git a/policy/src/policy/utils/path.rs b/crates/policy/src/policy/utils/path.rs similarity index 100% rename from policy/src/policy/utils/path.rs rename to crates/policy/src/policy/utils/path.rs diff --git a/policy/src/policy/utils/wildcard.rs b/crates/policy/src/policy/utils/wildcard.rs similarity index 100% rename from policy/src/policy/utils/wildcard.rs rename to crates/policy/src/policy/utils/wildcard.rs diff --git a/policy/src/service_type.rs b/crates/policy/src/service_type.rs similarity index 100% rename from policy/src/service_type.rs rename to crates/policy/src/service_type.rs diff --git a/policy/src/utils.rs b/crates/policy/src/utils.rs similarity index 100% rename from policy/src/utils.rs rename to crates/policy/src/utils.rs diff --git a/policy/tests/policy_is_allowed.rs b/crates/policy/tests/policy_is_allowed.rs similarity index 79% rename from policy/tests/policy_is_allowed.rs rename to crates/policy/tests/policy_is_allowed.rs index 674ea6ad..bbffc481 100644 --- a/policy/tests/policy_is_allowed.rs +++ b/crates/policy/tests/policy_is_allowed.rs @@ -12,11 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use policy::policy::ActionSet; -use policy::policy::Effect::*; -use policy::policy::action::Action; -use policy::policy::action::S3Action::*; -use policy::policy::*; +use rustfs_policy::policy::Effect::*; +use rustfs_policy::policy::action::S3Action::*; +use rustfs_policy::policy::*; use serde_json::Value; use std::collections::HashMap; use test_case::test_case; @@ -35,12 +33,12 @@ struct ArgsBuilder { } #[test_case( - policy::policy::Policy{ - version: policy::policy::DEFAULT_VERSION.into(), + Policy{ + version: DEFAULT_VERSION.into(), statements: vec![ - policy::policy::Statement{ + rustfs_policy::policy::Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(PutObjectAction), Action::S3Action(GetBucketLocationAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(PutObjectAction), rustfs_policy::policy::action::Action::S3Action(GetBucketLocationAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -61,7 +59,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(PutObjectAction), Action::S3Action(GetBucketLocationAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(PutObjectAction), rustfs_policy::policy::action::Action::S3Action(GetBucketLocationAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -88,7 +86,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(PutObjectAction), Action::S3Action(GetBucketLocationAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(PutObjectAction), rustfs_policy::policy::action::Action::S3Action(GetBucketLocationAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -110,7 +108,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(PutObjectAction), Action::S3Action(GetBucketLocationAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(PutObjectAction), rustfs_policy::policy::action::Action::S3Action(GetBucketLocationAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -131,7 +129,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(PutObjectAction), Action::S3Action(GetBucketLocationAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(PutObjectAction), rustfs_policy::policy::action::Action::S3Action(GetBucketLocationAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -159,7 +157,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(PutObjectAction), Action::S3Action(GetBucketLocationAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(PutObjectAction), rustfs_policy::policy::action::Action::S3Action(GetBucketLocationAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -181,7 +179,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -202,7 +200,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -230,7 +228,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -252,7 +250,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -273,7 +271,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -301,7 +299,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), ..Default::default() } @@ -323,7 +321,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -345,7 +343,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -374,7 +372,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -397,7 +395,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -419,7 +417,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -448,7 +446,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Allow, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -471,7 +469,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Deny, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -493,7 +491,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Deny, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -522,7 +520,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Deny, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -545,7 +543,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Deny, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -567,7 +565,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Deny, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() @@ -596,7 +594,7 @@ struct ArgsBuilder { statements: vec![ Statement{ effect: Deny, - actions: ActionSet(vec![Action::S3Action(GetObjectAction), Action::S3Action(PutObjectAction)].into_iter().collect()), + actions: ActionSet(vec![rustfs_policy::policy::action::Action::S3Action(GetObjectAction), rustfs_policy::policy::action::Action::S3Action(PutObjectAction)].into_iter().collect()), resources: ResourceSet(vec!["arn:aws:s3:::mybucket/myobject*".try_into().unwrap()].into_iter().collect()), conditions: serde_json::from_str(r#"{"IpAddress": {"aws:SourceIp": "192.168.1.0/24"}}"#).unwrap(), ..Default::default() diff --git a/common/protos/Cargo.toml b/crates/protos/Cargo.toml similarity index 79% rename from common/protos/Cargo.toml rename to crates/protos/Cargo.toml index dfc97a06..01068448 100644 --- a/common/protos/Cargo.toml +++ b/crates/protos/Cargo.toml @@ -13,7 +13,7 @@ # limitations under the License. [package] -name = "protos" +name = "rustfs-protos" version.workspace = true edition.workspace = true @@ -25,13 +25,8 @@ name = "gproto" path = "src/main.rs" [dependencies] -#async-backtrace = { workspace = true, optional = true } -common.workspace = true +rustfs-common.workspace = true flatbuffers = { workspace = true } prost = { workspace = true } -protobuf = { workspace = true } -tokio = { workspace = true } tonic = { workspace = true, features = ["transport"] } -tower = { workspace = true } -prost-build = { workspace = true } tonic-build = { workspace = true } diff --git a/common/protos/src/generated/flatbuffers_generated/mod.rs b/crates/protos/src/generated/flatbuffers_generated/mod.rs similarity index 100% rename from common/protos/src/generated/flatbuffers_generated/mod.rs rename to crates/protos/src/generated/flatbuffers_generated/mod.rs diff --git a/common/protos/src/generated/flatbuffers_generated/models.rs b/crates/protos/src/generated/flatbuffers_generated/models.rs similarity index 100% rename from common/protos/src/generated/flatbuffers_generated/models.rs rename to crates/protos/src/generated/flatbuffers_generated/models.rs diff --git a/common/protos/src/generated/mod.rs b/crates/protos/src/generated/mod.rs similarity index 100% rename from common/protos/src/generated/mod.rs rename to crates/protos/src/generated/mod.rs diff --git a/common/protos/src/generated/proto_gen/mod.rs b/crates/protos/src/generated/proto_gen/mod.rs similarity index 100% rename from common/protos/src/generated/proto_gen/mod.rs rename to crates/protos/src/generated/proto_gen/mod.rs diff --git a/common/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs similarity index 100% rename from common/protos/src/generated/proto_gen/node_service.rs rename to crates/protos/src/generated/proto_gen/node_service.rs diff --git a/common/protos/src/lib.rs b/crates/protos/src/lib.rs similarity index 97% rename from common/protos/src/lib.rs rename to crates/protos/src/lib.rs index 8cffdb18..2433eca2 100644 --- a/common/protos/src/lib.rs +++ b/crates/protos/src/lib.rs @@ -17,9 +17,9 @@ mod generated; use std::{error::Error, time::Duration}; -use common::globals::GLOBAL_Conn_Map; pub use generated::*; use proto_gen::node_service::node_service_client::NodeServiceClient; +use rustfs_common::globals::GLOBAL_Conn_Map; use tonic::{ Request, Status, metadata::MetadataValue, diff --git a/common/protos/src/main.rs b/crates/protos/src/main.rs similarity index 99% rename from common/protos/src/main.rs rename to crates/protos/src/main.rs index e8900eb5..5e48c4d5 100644 --- a/common/protos/src/main.rs +++ b/crates/protos/src/main.rs @@ -45,7 +45,7 @@ fn main() -> Result<(), AnyError> { } // path of proto file - let project_root_dir = env::current_dir()?.join("common/protos/src"); + let project_root_dir = env::current_dir()?.join(""); let proto_dir = project_root_dir.clone(); let proto_files = &["node.proto"]; let proto_out_dir = project_root_dir.join("generated").join("proto_gen"); diff --git a/common/protos/src/models.fbs b/crates/protos/src/models.fbs similarity index 100% rename from common/protos/src/models.fbs rename to crates/protos/src/models.fbs diff --git a/common/protos/src/node.proto b/crates/protos/src/node.proto similarity index 100% rename from common/protos/src/node.proto rename to crates/protos/src/node.proto diff --git a/crates/rio/Cargo.toml b/crates/rio/Cargo.toml index f4f65db2..1c3d806c 100644 --- a/crates/rio/Cargo.toml +++ b/crates/rio/Cargo.toml @@ -26,22 +26,18 @@ workspace = true [dependencies] tokio = { workspace = true, features = ["full"] } rand = { workspace = true } -md-5 = { workspace = true } http.workspace = true -aes-gcm = "0.10.3" -crc32fast = "1.4.2" +aes-gcm = { workspace = true } +crc32fast = { workspace = true } pin-project-lite.workspace = true -async-trait.workspace = true -base64-simd = "0.8.0" -hex-simd = "0.8.0" serde = { workspace = true } bytes.workspace = true reqwest.workspace = true tokio-util.workspace = true futures.workspace = true -rustfs-utils = {workspace = true, features= ["io","hash","compress"]} -byteorder.workspace = true +rustfs-utils = { workspace = true, features = ["io", "hash", "compress"] } serde_json.workspace = true +md-5 = { workspace = true } [dev-dependencies] criterion = { version = "0.5.1", features = ["async", "async_tokio", "tokio"] } diff --git a/s3select/api/Cargo.toml b/crates/s3select-api/Cargo.toml similarity index 85% rename from s3select/api/Cargo.toml rename to crates/s3select-api/Cargo.toml index 783e415d..93554009 100644 --- a/s3select/api/Cargo.toml +++ b/crates/s3select-api/Cargo.toml @@ -13,17 +13,20 @@ # limitations under the License. [package] -name = "api" +name = "rustfs-s3select-api" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true [dependencies] async-trait.workspace = true bytes.workspace = true chrono.workspace = true -common.workspace = true +rustfs-common.workspace = true datafusion = { workspace = true } -ecstore.workspace = true +rustfs-ecstore.workspace = true futures = { workspace = true } futures-core = { workspace = true } http.workspace = true diff --git a/s3select/api/src/lib.rs b/crates/s3select-api/src/lib.rs similarity index 100% rename from s3select/api/src/lib.rs rename to crates/s3select-api/src/lib.rs diff --git a/s3select/api/src/object_store.rs b/crates/s3select-api/src/object_store.rs similarity index 96% rename from s3select/api/src/object_store.rs rename to crates/s3select-api/src/object_store.rs index 68d54940..79594a4c 100644 --- a/s3select/api/src/object_store.rs +++ b/crates/s3select-api/src/object_store.rs @@ -15,13 +15,6 @@ use async_trait::async_trait; use bytes::Bytes; use chrono::Utc; -use common::DEFAULT_DELIMITER; -use ecstore::StorageAPI; -use ecstore::new_object_layer_fn; -use ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; -use ecstore::store::ECStore; -use ecstore::store_api::ObjectIO; -use ecstore::store_api::ObjectOptions; use futures::pin_mut; use futures::{Stream, StreamExt}; use futures_core::stream::BoxStream; @@ -40,6 +33,13 @@ use object_store::PutResult; use object_store::path::Path; use object_store::{Error as o_Error, Result}; use pin_project_lite::pin_project; +use rustfs_common::DEFAULT_DELIMITER; +use rustfs_ecstore::StorageAPI; +use rustfs_ecstore::new_object_layer_fn; +use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; +use rustfs_ecstore::store::ECStore; +use rustfs_ecstore::store_api::ObjectIO; +use rustfs_ecstore::store_api::ObjectOptions; use s3s::S3Result; use s3s::dto::SelectObjectContentInput; use s3s::s3_error; diff --git a/s3select/api/src/query/analyzer.rs b/crates/s3select-api/src/query/analyzer.rs similarity index 100% rename from s3select/api/src/query/analyzer.rs rename to crates/s3select-api/src/query/analyzer.rs diff --git a/s3select/api/src/query/ast.rs b/crates/s3select-api/src/query/ast.rs similarity index 100% rename from s3select/api/src/query/ast.rs rename to crates/s3select-api/src/query/ast.rs diff --git a/s3select/api/src/query/datasource/mod.rs b/crates/s3select-api/src/query/datasource/mod.rs similarity index 100% rename from s3select/api/src/query/datasource/mod.rs rename to crates/s3select-api/src/query/datasource/mod.rs diff --git a/s3select/api/src/query/dispatcher.rs b/crates/s3select-api/src/query/dispatcher.rs similarity index 100% rename from s3select/api/src/query/dispatcher.rs rename to crates/s3select-api/src/query/dispatcher.rs diff --git a/s3select/api/src/query/execution.rs b/crates/s3select-api/src/query/execution.rs similarity index 100% rename from s3select/api/src/query/execution.rs rename to crates/s3select-api/src/query/execution.rs diff --git a/s3select/api/src/query/function.rs b/crates/s3select-api/src/query/function.rs similarity index 100% rename from s3select/api/src/query/function.rs rename to crates/s3select-api/src/query/function.rs diff --git a/s3select/api/src/query/logical_planner.rs b/crates/s3select-api/src/query/logical_planner.rs similarity index 100% rename from s3select/api/src/query/logical_planner.rs rename to crates/s3select-api/src/query/logical_planner.rs diff --git a/s3select/api/src/query/mod.rs b/crates/s3select-api/src/query/mod.rs similarity index 100% rename from s3select/api/src/query/mod.rs rename to crates/s3select-api/src/query/mod.rs diff --git a/s3select/api/src/query/optimizer.rs b/crates/s3select-api/src/query/optimizer.rs similarity index 100% rename from s3select/api/src/query/optimizer.rs rename to crates/s3select-api/src/query/optimizer.rs diff --git a/s3select/api/src/query/parser.rs b/crates/s3select-api/src/query/parser.rs similarity index 100% rename from s3select/api/src/query/parser.rs rename to crates/s3select-api/src/query/parser.rs diff --git a/s3select/api/src/query/physical_planner.rs b/crates/s3select-api/src/query/physical_planner.rs similarity index 100% rename from s3select/api/src/query/physical_planner.rs rename to crates/s3select-api/src/query/physical_planner.rs diff --git a/s3select/api/src/query/scheduler.rs b/crates/s3select-api/src/query/scheduler.rs similarity index 100% rename from s3select/api/src/query/scheduler.rs rename to crates/s3select-api/src/query/scheduler.rs diff --git a/s3select/api/src/query/session.rs b/crates/s3select-api/src/query/session.rs similarity index 100% rename from s3select/api/src/query/session.rs rename to crates/s3select-api/src/query/session.rs diff --git a/s3select/api/src/server/dbms.rs b/crates/s3select-api/src/server/dbms.rs similarity index 100% rename from s3select/api/src/server/dbms.rs rename to crates/s3select-api/src/server/dbms.rs diff --git a/s3select/api/src/server/mod.rs b/crates/s3select-api/src/server/mod.rs similarity index 100% rename from s3select/api/src/server/mod.rs rename to crates/s3select-api/src/server/mod.rs diff --git a/s3select/query/Cargo.toml b/crates/s3select-query/Cargo.toml similarity index 86% rename from s3select/query/Cargo.toml rename to crates/s3select-query/Cargo.toml index 98cde279..1e6d8e27 100644 --- a/s3select/query/Cargo.toml +++ b/crates/s3select-query/Cargo.toml @@ -13,12 +13,15 @@ # limitations under the License. [package] -name = "query" +name = "rustfs-s3select-query" version.workspace = true edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true [dependencies] -api = { workspace = true } +rustfs-s3select-api = { workspace = true } async-recursion = { workspace = true } async-trait.workspace = true datafusion = { workspace = true } diff --git a/s3select/query/src/data_source/mod.rs b/crates/s3select-query/src/data_source/mod.rs similarity index 100% rename from s3select/query/src/data_source/mod.rs rename to crates/s3select-query/src/data_source/mod.rs diff --git a/s3select/query/src/data_source/table_source.rs b/crates/s3select-query/src/data_source/table_source.rs similarity index 100% rename from s3select/query/src/data_source/table_source.rs rename to crates/s3select-query/src/data_source/table_source.rs diff --git a/s3select/query/src/dispatcher/manager.rs b/crates/s3select-query/src/dispatcher/manager.rs similarity index 99% rename from s3select/query/src/dispatcher/manager.rs rename to crates/s3select-query/src/dispatcher/manager.rs index e840f0a2..1f73501a 100644 --- a/s3select/query/src/dispatcher/manager.rs +++ b/crates/s3select-query/src/dispatcher/manager.rs @@ -19,19 +19,6 @@ use std::{ task::{Context, Poll}, }; -use api::{ - QueryError, QueryResult, - query::{ - Query, - ast::ExtStatement, - dispatcher::QueryDispatcher, - execution::{Output, QueryStateMachine}, - function::FuncMetaManagerRef, - logical_planner::{LogicalPlanner, Plan}, - parser::Parser, - session::{SessionCtx, SessionCtxFactory}, - }, -}; use async_trait::async_trait; use datafusion::{ arrow::{ @@ -47,6 +34,19 @@ use datafusion::{ }; use futures::{Stream, StreamExt}; use lazy_static::lazy_static; +use rustfs_s3select_api::{ + QueryError, QueryResult, + query::{ + Query, + ast::ExtStatement, + dispatcher::QueryDispatcher, + execution::{Output, QueryStateMachine}, + function::FuncMetaManagerRef, + logical_planner::{LogicalPlanner, Plan}, + parser::Parser, + session::{SessionCtx, SessionCtxFactory}, + }, +}; use s3s::dto::{FileHeaderInfo, SelectObjectContentInput}; use crate::{ diff --git a/s3select/query/src/dispatcher/mod.rs b/crates/s3select-query/src/dispatcher/mod.rs similarity index 100% rename from s3select/query/src/dispatcher/mod.rs rename to crates/s3select-query/src/dispatcher/mod.rs diff --git a/s3select/query/src/execution/factory.rs b/crates/s3select-query/src/execution/factory.rs similarity index 98% rename from s3select/query/src/execution/factory.rs rename to crates/s3select-query/src/execution/factory.rs index 43deeebe..f21b154d 100644 --- a/s3select/query/src/execution/factory.rs +++ b/crates/s3select-query/src/execution/factory.rs @@ -14,7 +14,8 @@ use std::sync::Arc; -use api::{ +use async_trait::async_trait; +use rustfs_s3select_api::{ QueryError, query::{ execution::{QueryExecutionFactory, QueryExecutionRef, QueryStateMachineRef}, @@ -23,7 +24,6 @@ use api::{ scheduler::SchedulerRef, }, }; -use async_trait::async_trait; use super::query::SqlQueryExecution; diff --git a/s3select/query/src/execution/mod.rs b/crates/s3select-query/src/execution/mod.rs similarity index 100% rename from s3select/query/src/execution/mod.rs rename to crates/s3select-query/src/execution/mod.rs diff --git a/s3select/query/src/execution/query.rs b/crates/s3select-query/src/execution/query.rs similarity index 90% rename from s3select/query/src/execution/query.rs rename to crates/s3select-query/src/execution/query.rs index b49b4ba4..d48cf28d 100644 --- a/s3select/query/src/execution/query.rs +++ b/crates/s3select-query/src/execution/query.rs @@ -14,14 +14,14 @@ use std::sync::Arc; -use api::query::execution::{Output, QueryExecution, QueryStateMachineRef}; -use api::query::logical_planner::QueryPlan; -use api::query::optimizer::Optimizer; -use api::query::scheduler::SchedulerRef; -use api::{QueryError, QueryResult}; use async_trait::async_trait; use futures::stream::AbortHandle; use parking_lot::Mutex; +use rustfs_s3select_api::query::execution::{Output, QueryExecution, QueryStateMachineRef}; +use rustfs_s3select_api::query::logical_planner::QueryPlan; +use rustfs_s3select_api::query::optimizer::Optimizer; +use rustfs_s3select_api::query::scheduler::SchedulerRef; +use rustfs_s3select_api::{QueryError, QueryResult}; use tracing::debug; pub struct SqlQueryExecution { diff --git a/s3select/query/src/execution/scheduler/local.rs b/crates/s3select-query/src/execution/scheduler/local.rs similarity index 93% rename from s3select/query/src/execution/scheduler/local.rs rename to crates/s3select-query/src/execution/scheduler/local.rs index e5edd9da..fbc0d078 100644 --- a/s3select/query/src/execution/scheduler/local.rs +++ b/crates/s3select-query/src/execution/scheduler/local.rs @@ -14,11 +14,11 @@ use std::sync::Arc; -use api::query::scheduler::{ExecutionResults, Scheduler}; use async_trait::async_trait; use datafusion::error::DataFusionError; use datafusion::execution::context::TaskContext; use datafusion::physical_plan::{ExecutionPlan, execute_stream}; +use rustfs_s3select_api::query::scheduler::{ExecutionResults, Scheduler}; pub struct LocalScheduler {} diff --git a/s3select/query/src/execution/scheduler/mod.rs b/crates/s3select-query/src/execution/scheduler/mod.rs similarity index 100% rename from s3select/query/src/execution/scheduler/mod.rs rename to crates/s3select-query/src/execution/scheduler/mod.rs diff --git a/s3select/query/src/function/mod.rs b/crates/s3select-query/src/function/mod.rs similarity index 100% rename from s3select/query/src/function/mod.rs rename to crates/s3select-query/src/function/mod.rs diff --git a/s3select/query/src/function/simple_func_manager.rs b/crates/s3select-query/src/function/simple_func_manager.rs similarity index 97% rename from s3select/query/src/function/simple_func_manager.rs rename to crates/s3select-query/src/function/simple_func_manager.rs index 9bcafd26..7b1f63c0 100644 --- a/s3select/query/src/function/simple_func_manager.rs +++ b/crates/s3select-query/src/function/simple_func_manager.rs @@ -15,10 +15,10 @@ use std::collections::HashMap; use std::sync::Arc; -use api::query::function::FunctionMetadataManager; -use api::{QueryError, QueryResult}; use datafusion::execution::SessionStateDefaults; use datafusion::logical_expr::{AggregateUDF, ScalarUDF, WindowUDF}; +use rustfs_s3select_api::query::function::FunctionMetadataManager; +use rustfs_s3select_api::{QueryError, QueryResult}; use tracing::debug; pub type SimpleFunctionMetadataManagerRef = Arc; diff --git a/s3select/query/src/instance.rs b/crates/s3select-query/src/instance.rs similarity index 99% rename from s3select/query/src/instance.rs rename to crates/s3select-query/src/instance.rs index a1ca7e4b..403bfd25 100644 --- a/s3select/query/src/instance.rs +++ b/crates/s3select-query/src/instance.rs @@ -14,15 +14,15 @@ use std::sync::Arc; -use api::{ +use async_trait::async_trait; +use derive_builder::Builder; +use rustfs_s3select_api::{ QueryResult, query::{ Query, dispatcher::QueryDispatcher, execution::QueryStateMachineRef, logical_planner::Plan, session::SessionCtxFactory, }, server::dbms::{DatabaseManagerSystem, QueryHandle}, }; -use async_trait::async_trait; -use derive_builder::Builder; use s3s::dto::SelectObjectContentInput; use crate::{ @@ -111,11 +111,11 @@ pub async fn make_rustfsms(input: Arc, is_test: bool) mod tests { use std::sync::Arc; - use api::{ + use datafusion::{arrow::util::pretty, assert_batches_eq}; + use rustfs_s3select_api::{ query::{Context, Query}, server::dbms::DatabaseManagerSystem, }; - use datafusion::{arrow::util::pretty, assert_batches_eq}; use s3s::dto::{ CSVInput, CSVOutput, ExpressionType, FieldDelimiter, FileHeaderInfo, InputSerialization, OutputSerialization, RecordDelimiter, SelectObjectContentInput, SelectObjectContentRequest, diff --git a/s3select/query/src/lib.rs b/crates/s3select-query/src/lib.rs similarity index 100% rename from s3select/query/src/lib.rs rename to crates/s3select-query/src/lib.rs diff --git a/s3select/query/src/metadata/base_table.rs b/crates/s3select-query/src/metadata/base_table.rs similarity index 100% rename from s3select/query/src/metadata/base_table.rs rename to crates/s3select-query/src/metadata/base_table.rs diff --git a/s3select/query/src/metadata/mod.rs b/crates/s3select-query/src/metadata/mod.rs similarity index 98% rename from s3select/query/src/metadata/mod.rs rename to crates/s3select-query/src/metadata/mod.rs index 78bc2e05..707e4440 100644 --- a/s3select/query/src/metadata/mod.rs +++ b/crates/s3select-query/src/metadata/mod.rs @@ -14,7 +14,6 @@ use std::sync::Arc; -use api::query::{function::FuncMetaManagerRef, session::SessionCtx}; use async_trait::async_trait; use datafusion::arrow::datatypes::DataType; use datafusion::common::Result as DFResult; @@ -26,6 +25,7 @@ use datafusion::{ config::ConfigOptions, sql::{TableReference, planner::ContextProvider}, }; +use rustfs_s3select_api::query::{function::FuncMetaManagerRef, session::SessionCtx}; use crate::data_source::table_source::{TableHandle, TableSourceAdapter}; diff --git a/s3select/query/src/sql/analyzer.rs b/crates/s3select-query/src/sql/analyzer.rs similarity index 90% rename from s3select/query/src/sql/analyzer.rs rename to crates/s3select-query/src/sql/analyzer.rs index 704b7e2f..7d0e0a64 100644 --- a/s3select/query/src/sql/analyzer.rs +++ b/crates/s3select-query/src/sql/analyzer.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use api::QueryResult; -use api::query::analyzer::Analyzer; -use api::query::session::SessionCtx; use datafusion::logical_expr::LogicalPlan; use datafusion::optimizer::analyzer::Analyzer as DFAnalyzer; +use rustfs_s3select_api::QueryResult; +use rustfs_s3select_api::query::analyzer::Analyzer; +use rustfs_s3select_api::query::session::SessionCtx; pub struct DefaultAnalyzer { inner: DFAnalyzer, diff --git a/s3select/query/src/sql/dialect.rs b/crates/s3select-query/src/sql/dialect.rs similarity index 100% rename from s3select/query/src/sql/dialect.rs rename to crates/s3select-query/src/sql/dialect.rs diff --git a/s3select/query/src/sql/logical/mod.rs b/crates/s3select-query/src/sql/logical/mod.rs similarity index 100% rename from s3select/query/src/sql/logical/mod.rs rename to crates/s3select-query/src/sql/logical/mod.rs diff --git a/s3select/query/src/sql/logical/optimizer.rs b/crates/s3select-query/src/sql/logical/optimizer.rs similarity index 99% rename from s3select/query/src/sql/logical/optimizer.rs rename to crates/s3select-query/src/sql/logical/optimizer.rs index a0a6d749..603c738a 100644 --- a/s3select/query/src/sql/logical/optimizer.rs +++ b/crates/s3select-query/src/sql/logical/optimizer.rs @@ -14,10 +14,6 @@ use std::sync::Arc; -use api::{ - QueryResult, - query::{analyzer::AnalyzerRef, logical_planner::QueryPlan, session::SessionCtx}, -}; use datafusion::{ execution::SessionStateBuilder, logical_expr::LogicalPlan, @@ -33,6 +29,10 @@ use datafusion::{ unwrap_cast_in_comparison::UnwrapCastInComparison, }, }; +use rustfs_s3select_api::{ + QueryResult, + query::{analyzer::AnalyzerRef, logical_planner::QueryPlan, session::SessionCtx}, +}; use tracing::debug; use crate::sql::analyzer::DefaultAnalyzer; diff --git a/s3select/query/src/sql/logical/planner.rs b/crates/s3select-query/src/sql/logical/planner.rs similarity index 100% rename from s3select/query/src/sql/logical/planner.rs rename to crates/s3select-query/src/sql/logical/planner.rs diff --git a/s3select/query/src/sql/mod.rs b/crates/s3select-query/src/sql/mod.rs similarity index 100% rename from s3select/query/src/sql/mod.rs rename to crates/s3select-query/src/sql/mod.rs diff --git a/s3select/query/src/sql/optimizer.rs b/crates/s3select-query/src/sql/optimizer.rs similarity index 99% rename from s3select/query/src/sql/optimizer.rs rename to crates/s3select-query/src/sql/optimizer.rs index c22a8c7d..f4c68600 100644 --- a/s3select/query/src/sql/optimizer.rs +++ b/crates/s3select-query/src/sql/optimizer.rs @@ -14,12 +14,12 @@ use std::sync::Arc; -use api::{ +use async_trait::async_trait; +use datafusion::physical_plan::{ExecutionPlan, displayable}; +use rustfs_s3select_api::{ QueryResult, query::{logical_planner::QueryPlan, optimizer::Optimizer, physical_planner::PhysicalPlanner, session::SessionCtx}, }; -use async_trait::async_trait; -use datafusion::physical_plan::{ExecutionPlan, displayable}; use tracing::debug; use super::{ diff --git a/s3select/query/src/sql/parser.rs b/crates/s3select-query/src/sql/parser.rs similarity index 98% rename from s3select/query/src/sql/parser.rs rename to crates/s3select-query/src/sql/parser.rs index 453d55f3..4f705ea8 100644 --- a/s3select/query/src/sql/parser.rs +++ b/crates/s3select-query/src/sql/parser.rs @@ -14,15 +14,15 @@ use std::{collections::VecDeque, fmt::Display}; -use api::{ - ParserSnafu, - query::{ast::ExtStatement, parser::Parser as RustFsParser}, -}; use datafusion::sql::sqlparser::{ dialect::Dialect, parser::{Parser, ParserError}, tokenizer::{Token, Tokenizer}, }; +use rustfs_s3select_api::{ + ParserSnafu, + query::{ast::ExtStatement, parser::Parser as RustFsParser}, +}; use snafu::ResultExt; use super::dialect::RustFsDialect; @@ -40,7 +40,7 @@ macro_rules! parser_err { pub struct DefaultParser {} impl RustFsParser for DefaultParser { - fn parse(&self, sql: &str) -> api::QueryResult> { + fn parse(&self, sql: &str) -> rustfs_s3select_api::QueryResult> { ExtParser::parse_sql(sql).context(ParserSnafu) } } @@ -108,7 +108,7 @@ impl<'a> ExtParser<'a> { #[cfg(test)] mod tests { use super::*; - use api::query::ast::ExtStatement; + use rustfs_s3select_api::query::ast::ExtStatement; #[test] fn test_default_parser_creation() { diff --git a/s3select/query/src/sql/physical/mod.rs b/crates/s3select-query/src/sql/physical/mod.rs similarity index 100% rename from s3select/query/src/sql/physical/mod.rs rename to crates/s3select-query/src/sql/physical/mod.rs diff --git a/s3select/query/src/sql/physical/optimizer.rs b/crates/s3select-query/src/sql/physical/optimizer.rs similarity index 91% rename from s3select/query/src/sql/physical/optimizer.rs rename to crates/s3select-query/src/sql/physical/optimizer.rs index dec17641..7cbb4ca2 100644 --- a/s3select/query/src/sql/physical/optimizer.rs +++ b/crates/s3select-query/src/sql/physical/optimizer.rs @@ -14,10 +14,10 @@ use std::sync::Arc; -use api::QueryResult; -use api::query::session::SessionCtx; use datafusion::physical_optimizer::PhysicalOptimizerRule; use datafusion::physical_plan::ExecutionPlan; +use rustfs_s3select_api::QueryResult; +use rustfs_s3select_api::query::session::SessionCtx; pub trait PhysicalOptimizer { fn optimize(&self, plan: Arc, session: &SessionCtx) -> QueryResult>; diff --git a/s3select/query/src/sql/physical/planner.rs b/crates/s3select-query/src/sql/physical/planner.rs similarity index 96% rename from s3select/query/src/sql/physical/planner.rs rename to crates/s3select-query/src/sql/physical/planner.rs index 26927a37..1f49f4b1 100644 --- a/s3select/query/src/sql/physical/planner.rs +++ b/crates/s3select-query/src/sql/physical/planner.rs @@ -14,9 +14,6 @@ use std::sync::Arc; -use api::QueryResult; -use api::query::physical_planner::PhysicalPlanner; -use api::query::session::SessionCtx; use async_trait::async_trait; use datafusion::execution::SessionStateBuilder; use datafusion::logical_expr::LogicalPlan; @@ -28,6 +25,9 @@ use datafusion::physical_plan::ExecutionPlan; use datafusion::physical_planner::{ DefaultPhysicalPlanner as DFDefaultPhysicalPlanner, ExtensionPlanner, PhysicalPlanner as DFPhysicalPlanner, }; +use rustfs_s3select_api::QueryResult; +use rustfs_s3select_api::query::physical_planner::PhysicalPlanner; +use rustfs_s3select_api::query::session::SessionCtx; use super::optimizer::PhysicalOptimizer; diff --git a/s3select/query/src/sql/planner.rs b/crates/s3select-query/src/sql/planner.rs similarity index 98% rename from s3select/query/src/sql/planner.rs rename to crates/s3select-query/src/sql/planner.rs index 4cc5d8e2..03d96ac5 100644 --- a/s3select/query/src/sql/planner.rs +++ b/crates/s3select-query/src/sql/planner.rs @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -use api::{ +use async_recursion::async_recursion; +use async_trait::async_trait; +use datafusion::sql::{planner::SqlToRel, sqlparser::ast::Statement}; +use rustfs_s3select_api::{ QueryError, QueryResult, query::{ ast::ExtStatement, @@ -20,9 +23,6 @@ use api::{ session::SessionCtx, }, }; -use async_recursion::async_recursion; -use async_trait::async_trait; -use datafusion::sql::{planner::SqlToRel, sqlparser::ast::Statement}; use crate::metadata::ContextProviderExtension; diff --git a/crates/signer/Cargo.toml b/crates/signer/Cargo.toml index 42831a33..0c878fd1 100644 --- a/crates/signer/Cargo.toml +++ b/crates/signer/Cargo.toml @@ -27,13 +27,10 @@ bytes = { workspace = true } http.workspace = true time.workspace = true hyper.workspace = true -serde.workspace = true serde_urlencoded.workspace = true -rustfs-utils = {workspace = true, features=["full"]} +rustfs-utils = { workspace = true, features = ["full"] } [dev-dependencies] -tempfile = { workspace = true } -rand = { workspace = true } [lints] workspace = true diff --git a/crates/utils/Cargo.toml b/crates/utils/Cargo.toml index 17384238..f62d5712 100644 --- a/crates/utils/Cargo.toml +++ b/crates/utils/Cargo.toml @@ -53,7 +53,6 @@ transform-stream = { workspace = true, optional = true } bytes = { workspace = true, optional = true } sysinfo = { workspace = true, optional = true } hyper-util = { workspace = true, optional = true } -common.workspace = true sha1 = { workspace = true, optional = true } sha2 = { workspace = true, optional = true } hmac = { workspace = true, optional = true } @@ -81,7 +80,7 @@ notify = ["dep:hyper", "dep:s3s"] # file system notification features compress = ["dep:flate2", "dep:brotli", "dep:snap", "dep:lz4", "dep:zstd"] string = ["dep:regex", "dep:lazy_static", "dep:rand"] crypto = ["dep:base64-simd", "dep:hex-simd", "dep:hmac", "dep:hyper", "dep:sha1"] -hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd","dep:base64-simd"] +hash = ["dep:highway", "dep:md-5", "dep:sha2", "dep:blake3", "dep:serde", "dep:siphasher", "dep:hex-simd", "dep:base64-simd"] os = ["dep:nix", "dep:tempfile", "winapi"] # operating system utilities integration = [] # integration test features sys = ["dep:sysinfo"] # system information features diff --git a/common/workers/Cargo.toml b/crates/workers/Cargo.toml similarity index 95% rename from common/workers/Cargo.toml rename to crates/workers/Cargo.toml index 4c999189..0ffd3d30 100644 --- a/common/workers/Cargo.toml +++ b/crates/workers/Cargo.toml @@ -13,7 +13,7 @@ # limitations under the License. [package] -name = "workers" +name = "rustfs-workers" edition.workspace = true license.workspace = true repository.workspace = true @@ -24,6 +24,5 @@ version.workspace = true workspace = true [dependencies] -common.workspace = true tokio.workspace = true tracing.workspace = true diff --git a/common/workers/src/lib.rs b/crates/workers/src/lib.rs similarity index 100% rename from common/workers/src/lib.rs rename to crates/workers/src/lib.rs diff --git a/common/workers/src/workers.rs b/crates/workers/src/workers.rs similarity index 100% rename from common/workers/src/workers.rs rename to crates/workers/src/workers.rs diff --git a/crates/zip/Cargo.toml b/crates/zip/Cargo.toml index 5c5b6324..19d144ef 100644 --- a/crates/zip/Cargo.toml +++ b/crates/zip/Cargo.toml @@ -22,7 +22,7 @@ version.workspace = true [dependencies] -async-compression = { version = "0.4.0", features = [ +async-compression = { workspace = true, features = [ "tokio", "bzip2", "gzip", @@ -30,12 +30,9 @@ async-compression = { version = "0.4.0", features = [ "zstd", "xz", ] } -async_zip = { version = "0.0.17", features = ["tokio"] } -zip = { workspace = true } tokio = { workspace = true, features = ["full"] } tokio-stream = { workspace = true } tokio-tar = { workspace = true } -xz2 = { version = "0.1", optional = true, features = ["static"] } [lints] diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 82af0cb2..11f951f2 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -31,47 +31,37 @@ workspace = true [dependencies] rustfs-zip = { workspace = true } tokio-tar = { workspace = true } -madmin = { workspace = true } -api = { workspace = true } -appauth = { workspace = true } +rustfs-madmin = { workspace = true } +rustfs-s3select-api = { workspace = true } +rustfs-appauth = { workspace = true } +rustfs-ecstore = { workspace = true } +rustfs-policy = { workspace = true } +rustfs-common = { workspace = true } atoi = { workspace = true } atomic_enum = { workspace = true } -aws-sdk-s3 = { workspace = true } axum.workspace = true axum-extra = { workspace = true } axum-server = { workspace = true } -async-trait.workspace = true -bytes.workspace = true +async-trait = { workspace = true } +bytes = { workspace = true } chrono = { workspace = true } -clap.workspace = true -crypto = { workspace = true } +clap = { workspace = true } datafusion = { workspace = true } -common.workspace = true const-str = { workspace = true } -ecstore.workspace = true -policy.workspace = true -flatbuffers.workspace = true futures.workspace = true -futures-util.workspace = true hyper.workspace = true hyper-util.workspace = true http.workspace = true http-body.workspace = true -iam = { workspace = true } -include_dir = { workspace = true } -jsonwebtoken = { workspace = true } -lock.workspace = true +rustfs-iam = { workspace = true } matchit = { workspace = true } -mime.workspace = true mime_guess = { workspace = true } opentelemetry = { workspace = true } percent-encoding = { workspace = true } pin-project-lite.workspace = true -protos.workspace = true -query = { workspace = true } -regex = { workspace = true } +rustfs-protos.workspace = true +rustfs-s3select-query = { workspace = true } reqwest = { workspace = true } -rmp-serde.workspace = true rustfs-config = { workspace = true, features = ["constants", "notify"] } rustfs-notify = { workspace = true } rustfs-obs = { workspace = true } @@ -99,7 +89,6 @@ lazy_static.workspace = true tokio-stream.workspace = true tonic = { workspace = true } tower.workspace = true -transform-stream.workspace = true tower-http = { workspace = true, features = [ "trace", "compression-deflate", @@ -110,9 +99,6 @@ urlencoding = { workspace = true } uuid = { workspace = true } rustfs-filemeta.workspace = true rustfs-rio.workspace = true -base64 = { workspace = true } -hmac = { workspace = true } -sha2 = { workspace = true } zip = { workspace = true } [target.'cfg(target_os = "linux")'.dependencies] @@ -122,22 +108,13 @@ libsystemd.workspace = true tikv-jemallocator = "0.6" [build-dependencies] -prost-build.workspace = true -tonic-build.workspace = true http.workspace = true -bytes.workspace = true futures.workspace = true -futures-util.workspace = true -# uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] } -ecstore = { workspace = true } -s3s.workspace = true +s3s = { workspace = true } clap = { workspace = true } hyper-util = { workspace = true, features = [ "tokio", "server-auto", "server-graceful", ] } -transform-stream = { workspace = true } -netif = "0.1.6" shadow-rs = { workspace = true, features = ["build"] } -# pin-utils = "0.1.0" diff --git a/rustfs/src/admin/handlers.rs b/rustfs/src/admin/handlers.rs index 6797169b..d96dadd8 100644 --- a/rustfs/src/admin/handlers.rs +++ b/rustfs/src/admin/handlers.rs @@ -18,39 +18,38 @@ use crate::auth::get_condition_values; use crate::auth::get_session_token; use crate::error::ApiError; use bytes::Bytes; -use ecstore::admin_server_info::get_server_info; -use ecstore::bucket::metadata_sys::{self, get_replication_config}; -use ecstore::bucket::target::BucketTarget; -use ecstore::bucket::versioning_sys::BucketVersioningSys; -use ecstore::cmd::bucket_targets::{self, GLOBAL_Bucket_Target_Sys}; -use ecstore::error::StorageError; -use ecstore::global::GLOBAL_ALlHealState; -use ecstore::global::get_global_action_cred; -use ecstore::heal::data_usage::load_data_usage_from_backend; -use ecstore::heal::heal_commands::HealOpts; -use ecstore::heal::heal_ops::new_heal_sequence; -use ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics}; -use ecstore::new_object_layer_fn; -use ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free}; -use ecstore::store::is_valid_object_prefix; -use ecstore::store_api::BucketOptions; -use ecstore::store_api::StorageAPI; -use ecstore::store_utils::is_reserved_or_invalid_bucket; use futures::{Stream, StreamExt}; use http::{HeaderMap, Uri}; use hyper::StatusCode; -use iam::store::MappedPolicy; -use rustfs_utils::path::path_join; -// use lazy_static::lazy_static; -use madmin::metrics::RealtimeMetrics; -use madmin::utils::parse_duration; use matchit::Params; use percent_encoding::{AsciiSet, CONTROLS, percent_encode}; -use policy::policy::Args; -use policy::policy::BucketPolicy; -use policy::policy::action::Action; -use policy::policy::action::S3Action; -use policy::policy::default::DEFAULT_POLICIES; +use rustfs_ecstore::admin_server_info::get_server_info; +use rustfs_ecstore::bucket::metadata_sys::{self, get_replication_config}; +use rustfs_ecstore::bucket::target::BucketTarget; +use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys; +use rustfs_ecstore::cmd::bucket_targets::{self, GLOBAL_Bucket_Target_Sys}; +use rustfs_ecstore::error::StorageError; +use rustfs_ecstore::global::GLOBAL_ALlHealState; +use rustfs_ecstore::global::get_global_action_cred; +use rustfs_ecstore::heal::data_usage::load_data_usage_from_backend; +use rustfs_ecstore::heal::heal_commands::HealOpts; +use rustfs_ecstore::heal::heal_ops::new_heal_sequence; +use rustfs_ecstore::metrics_realtime::{CollectMetricsOpts, MetricType, collect_local_metrics}; +use rustfs_ecstore::new_object_layer_fn; +use rustfs_ecstore::pools::{get_total_usable_capacity, get_total_usable_capacity_free}; +use rustfs_ecstore::store::is_valid_object_prefix; +use rustfs_ecstore::store_api::BucketOptions; +use rustfs_ecstore::store_api::StorageAPI; +use rustfs_ecstore::store_utils::is_reserved_or_invalid_bucket; +use rustfs_iam::store::MappedPolicy; +use rustfs_madmin::metrics::RealtimeMetrics; +use rustfs_madmin::utils::parse_duration; +use rustfs_policy::policy::Args; +use rustfs_policy::policy::BucketPolicy; +use rustfs_policy::policy::action::Action; +use rustfs_policy::policy::action::S3Action; +use rustfs_policy::policy::default::DEFAULT_POLICIES; +use rustfs_utils::path::path_join; use s3s::header::CONTENT_TYPE; use s3s::stream::{ByteStream, DynByteStream}; use s3s::{Body, S3Error, S3Request, S3Response, S3Result, s3_error}; @@ -87,7 +86,7 @@ use urlencoding::decode; #[serde(rename_all = "PascalCase", default)] pub struct AccountInfo { pub account_name: String, - pub server: madmin::BackendInfo, + pub server: rustfs_madmin::BackendInfo, pub policy: BucketPolicy, } @@ -106,7 +105,9 @@ impl Operation for AccountInfoHandler { let (cred, owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; let default_claims = HashMap::new(); let claims = cred.claims.as_ref().unwrap_or(&default_claims); @@ -211,7 +212,7 @@ impl Operation for AccountInfoHandler { )); }; - let mut effective_policy: policy::policy::Policy = Default::default(); + let mut effective_policy: rustfs_policy::policy::Policy = Default::default(); if account_name == admin_cred.access_key { for (name, p) in DEFAULT_POLICIES.iter() { @@ -240,7 +241,7 @@ impl Operation for AccountInfoHandler { let policy_str = serde_json::to_string(&effective_policy) .map_err(|_e| S3Error::with_message(S3ErrorCode::InternalError, "parse policy failed"))?; - let mut account_info = madmin::AccountInfo { + let mut account_info = rustfs_madmin::AccountInfo { account_name, server: store.backend_info().await, policy: serde_json::Value::String(policy_str), @@ -261,15 +262,15 @@ impl Operation for AccountInfoHandler { if rd || wr { // TODO: BucketQuotaSys // TODO: other attributes - account_info.buckets.push(madmin::BucketAccessInfo { + account_info.buckets.push(rustfs_madmin::BucketAccessInfo { name: bucket.name.clone(), - details: Some(madmin::BucketDetails { + details: Some(rustfs_madmin::BucketDetails { versioning: BucketVersioningSys::enabled(bucket.name.as_str()).await, versioning_suspended: BucketVersioningSys::suspended(bucket.name.as_str()).await, ..Default::default() }), created: bucket.created, - access: madmin::AccountAccess { read: rd, write: wr }, + access: rustfs_madmin::AccountAccess { read: rd, write: wr }, ..Default::default() }); } @@ -832,9 +833,9 @@ impl Operation for SetRemoteTargetHandler { }; // let binfo:BucketInfo = store - // .get_bucket_info(bucket, &ecstore::store_api::BucketOptions::default()).await; + // .get_bucket_info(bucket, &rustfs_ecstore::store_api::BucketOptions::default()).await; match store - .get_bucket_info(bucket, &ecstore::store_api::BucketOptions::default()) + .get_bucket_info(bucket, &rustfs_ecstore::store_api::BucketOptions::default()) .await { Ok(info) => { @@ -949,7 +950,7 @@ impl Operation for ListRemoteTargetHandler { }; match store - .get_bucket_info(bucket, &ecstore::store_api::BucketOptions::default()) + .get_bucket_info(bucket, &rustfs_ecstore::store_api::BucketOptions::default()) .await { Ok(info) => { @@ -1047,7 +1048,7 @@ impl Operation for RemoveRemoteTargetHandler { #[cfg(test)] mod test { - use ecstore::heal::heal_commands::HealOpts; + use rustfs_ecstore::heal::heal_commands::HealOpts; #[ignore] // FIXME: failed in github actions #[test] diff --git a/rustfs/src/admin/handlers/bucket_meta.rs b/rustfs/src/admin/handlers/bucket_meta.rs index e27c828f..074349bc 100644 --- a/rustfs/src/admin/handlers/bucket_meta.rs +++ b/rustfs/src/admin/handlers/bucket_meta.rs @@ -22,7 +22,9 @@ use crate::{ auth::{check_key_valid, get_session_token}, }; -use ecstore::{ +use http::{HeaderMap, StatusCode}; +use matchit::Params; +use rustfs_ecstore::{ StorageAPI, bucket::{ metadata::{ @@ -38,13 +40,11 @@ use ecstore::{ new_object_layer_fn, store_api::BucketOptions, }; -use ecstore::{ +use rustfs_ecstore::{ bucket::utils::{deserialize, serialize}, store_api::MakeBucketOptions, }; -use http::{HeaderMap, StatusCode}; -use matchit::Params; -use policy::policy::BucketPolicy; +use rustfs_policy::policy::BucketPolicy; use rustfs_utils::path::{SLASH_SEPARATOR, path_join_buf}; use s3s::{ Body, S3Request, S3Response, S3Result, @@ -125,15 +125,16 @@ impl Operation for ExportBucketMetadata { let conf_path = path_join_buf(&[bucket.name.as_str(), conf]); match conf { BUCKET_POLICY_CONFIG => { - let config: policy::policy::BucketPolicy = match metadata_sys::get_bucket_policy(&bucket.name).await { - Ok((res, _)) => res, - Err(e) => { - if e == StorageError::ConfigNotFound { - continue; + let config: rustfs_policy::policy::BucketPolicy = + match metadata_sys::get_bucket_policy(&bucket.name).await { + Ok((res, _)) => res, + Err(e) => { + if e == StorageError::ConfigNotFound { + continue; + } + return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); } - return Err(s3_error!(InternalError, "get bucket metadata failed: {e}")); - } - }; + }; let config_json = serde_json::to_vec(&config).map_err(|e| s3_error!(InternalError, "serialize config failed: {e}"))?; zip_writer diff --git a/rustfs/src/admin/handlers/event.rs b/rustfs/src/admin/handlers/event.rs index 1e07655e..2ac2f62d 100644 --- a/rustfs/src/admin/handlers/event.rs +++ b/rustfs/src/admin/handlers/event.rs @@ -74,14 +74,17 @@ impl Operation for SetNotificationTarget { let mut kvs_map: HashMap = serde_json::from_slice(&body) .map_err(|e| s3_error!(InvalidArgument, "invalid json body for target config: {}", e))?; // If there is an enable key, add an enable key value to "on" - if !kvs_map.contains_key(ecstore::config::ENABLE_KEY) { - kvs_map.insert(ecstore::config::ENABLE_KEY.to_string(), ecstore::config::ENABLE_ON.to_string()); + if !kvs_map.contains_key(rustfs_ecstore::config::ENABLE_KEY) { + kvs_map.insert( + rustfs_ecstore::config::ENABLE_KEY.to_string(), + rustfs_ecstore::config::ENABLE_ON.to_string(), + ); } - let kvs = ecstore::config::KVS( + let kvs = rustfs_ecstore::config::KVS( kvs_map .into_iter() - .map(|(key, value)| ecstore::config::KV { + .map(|(key, value)| rustfs_ecstore::config::KV { key, value, hidden_if_empty: false, // Set a default value diff --git a/rustfs/src/admin/handlers/group.rs b/rustfs/src/admin/handlers/group.rs index 457ae5fc..0fc4f6b5 100644 --- a/rustfs/src/admin/handlers/group.rs +++ b/rustfs/src/admin/handlers/group.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::global::get_global_action_cred; use http::{HeaderMap, StatusCode}; -use iam::error::{is_err_no_such_group, is_err_no_such_user}; -use madmin::GroupAddRemove; use matchit::Params; +use rustfs_ecstore::global::get_global_action_cred; +use rustfs_iam::error::{is_err_no_such_group, is_err_no_such_user}; +use rustfs_madmin::GroupAddRemove; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde::Deserialize; use serde_urlencoded::from_bytes; @@ -36,7 +36,7 @@ impl Operation for ListGroups { async fn call(&self, _req: S3Request, _params: Params<'_, '_>) -> S3Result> { warn!("handle ListGroups"); - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let groups = iam_store.list_groups().await.map_err(|e| { warn!("list groups failed, e: {:?}", e); @@ -67,7 +67,7 @@ impl Operation for GetGroup { GroupQuery::default() } }; - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let g = iam_store.get_group_description(&query.group).await.map_err(|e| { warn!("get group failed, e: {:?}", e); @@ -103,7 +103,7 @@ impl Operation for SetGroupStatus { return Err(s3_error!(InvalidArgument, "group is required")); } - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; if let Some(status) = query.status { match status.as_str() { @@ -154,7 +154,7 @@ impl Operation for UpdateGroupMembers { warn!("UpdateGroupMembers args {:?}", args); - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; for member in args.members.iter() { match iam_store.is_temp_user(member).await { diff --git a/rustfs/src/admin/handlers/policys.rs b/rustfs/src/admin/handlers/policys.rs index f4c6db46..dd513a1e 100644 --- a/rustfs/src/admin/handlers/policys.rs +++ b/rustfs/src/admin/handlers/policys.rs @@ -13,12 +13,12 @@ // limitations under the License. use crate::admin::{router::Operation, utils::has_space_be}; -use ecstore::global::get_global_action_cred; use http::{HeaderMap, StatusCode}; -use iam::error::is_err_no_such_user; -use iam::store::MappedPolicy; use matchit::Params; -use policy::policy::Policy; +use rustfs_ecstore::global::get_global_action_cred; +use rustfs_iam::error::is_err_no_such_user; +use rustfs_iam::store::MappedPolicy; +use rustfs_policy::policy::Policy; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde::Deserialize; use serde_urlencoded::from_bytes; @@ -46,7 +46,7 @@ impl Operation for ListCannedPolicies { } }; - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let policies = iam_store.list_polices(&query.bucket).await.map_err(|e| { warn!("list policies failed, e: {:?}", e); @@ -114,7 +114,7 @@ impl Operation for AddCannedPolicy { return Err(s3_error!(InvalidRequest, "policy version is empty")); } - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; iam_store.set_policy(&query.name, policy).await.map_err(|e| { warn!("set policy failed, e: {:?}", e); @@ -153,7 +153,7 @@ impl Operation for InfoCannedPolicy { return Err(s3_error!(InvalidArgument, "too many policies")); } - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; let pd = iam_store.info_policy(&query.name).await.map_err(|e| { warn!("info policy failed, e: {:?}", e); @@ -189,7 +189,7 @@ impl Operation for RemoveCannedPolicy { return Err(s3_error!(InvalidArgument, "policy name is empty")); } - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; iam_store.delete_policy(&query.name, true).await.map_err(|e| { warn!("delete policy failed, e: {:?}", e); @@ -237,7 +237,7 @@ impl Operation for SetPolicyForUserOrGroup { return Err(s3_error!(InvalidArgument, "user or group is empty")); } - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { return Err(s3_error!(InternalError, "iam not init")) }; if !query.is_group { match iam_store.is_temp_user(&query.user_or_group).await { @@ -275,7 +275,7 @@ impl Operation for SetPolicyForUserOrGroup { } iam_store - .policy_db_set(&query.user_or_group, iam::store::UserType::Reg, query.is_group, &query.policy_name) + .policy_db_set(&query.user_or_group, rustfs_iam::store::UserType::Reg, query.is_group, &query.policy_name) .await .map_err(|e| { warn!("policy db set failed, e: {:?}", e); diff --git a/rustfs/src/admin/handlers/pools.rs b/rustfs/src/admin/handlers/pools.rs index b3286cbe..6de19960 100644 --- a/rustfs/src/admin/handlers/pools.rs +++ b/rustfs/src/admin/handlers/pools.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::{GLOBAL_Endpoints, new_object_layer_fn}; use http::{HeaderMap, StatusCode}; use matchit::Params; +use rustfs_ecstore::{GLOBAL_Endpoints, new_object_layer_fn}; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde::Deserialize; use serde_urlencoded::from_bytes; diff --git a/rustfs/src/admin/handlers/rebalance.rs b/rustfs/src/admin/handlers/rebalance.rs index ee0c032f..1999392d 100644 --- a/rustfs/src/admin/handlers/rebalance.rs +++ b/rustfs/src/admin/handlers/rebalance.rs @@ -12,7 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::{ +use http::{HeaderMap, StatusCode}; +use matchit::Params; +use rustfs_ecstore::{ StorageAPI, error::StorageError, new_object_layer_fn, @@ -20,8 +22,6 @@ use ecstore::{ rebalance::{DiskStat, RebalSaveOpt}, store_api::BucketOptions, }; -use http::{HeaderMap, StatusCode}; -use matchit::Params; use s3s::{Body, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde::{Deserialize, Serialize}; use std::time::Duration; @@ -29,7 +29,7 @@ use time::OffsetDateTime; use tracing::warn; use crate::admin::router::Operation; -use ecstore::rebalance::RebalanceMeta; +use rustfs_ecstore::rebalance::RebalanceMeta; #[derive(Debug, Clone, Deserialize, Serialize)] pub struct RebalanceResp { diff --git a/rustfs/src/admin/handlers/service_account.rs b/rustfs/src/admin/handlers/service_account.rs index ab29d458..f8f28603 100644 --- a/rustfs/src/admin/handlers/service_account.rs +++ b/rustfs/src/admin/handlers/service_account.rs @@ -15,18 +15,18 @@ use crate::admin::utils::has_space_be; use crate::auth::{get_condition_values, get_session_token}; use crate::{admin::router::Operation, auth::check_key_valid}; -use ecstore::global::get_global_action_cred; use http::HeaderMap; use hyper::StatusCode; -use iam::error::is_err_no_such_service_account; -use iam::sys::{NewServiceAccountOpts, UpdateServiceAccountOpts}; -use madmin::{ +use matchit::Params; +use rustfs_ecstore::global::get_global_action_cred; +use rustfs_iam::error::is_err_no_such_service_account; +use rustfs_iam::sys::{NewServiceAccountOpts, UpdateServiceAccountOpts}; +use rustfs_madmin::{ AddServiceAccountReq, AddServiceAccountResp, Credentials, InfoServiceAccountResp, ListServiceAccountsResp, ServiceAccountInfo, UpdateServiceAccountReq, }; -use matchit::Params; -use policy::policy::action::{Action, AdminAction}; -use policy::policy::{Args, Policy}; +use rustfs_policy::policy::action::{Action, AdminAction}; +use rustfs_policy::policy::{Args, Policy}; use s3s::S3ErrorCode::InvalidRequest; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde::Deserialize; @@ -102,7 +102,9 @@ impl Operation for AddServiceAccount { req_is_derived_cred = true; } - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; let deny_only = cred.access_key == target_user || cred.parent_user == target_user; @@ -222,7 +224,9 @@ impl Operation for UpdateServiceAccount { let access_key = query.access_key; - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; // let svc_account = iam_store.get_service_account(&access_key).await.map_err(|e| { // debug!("get service account failed, e: {:?}", e); @@ -329,7 +333,9 @@ impl Operation for InfoServiceAccount { let access_key = query.access_key; - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; let (svc_account, session_policy) = iam_store.get_service_account(&access_key).await.map_err(|e| { debug!("get service account failed, e: {:?}", e); @@ -466,7 +472,9 @@ impl Operation for ListServiceAccount { // cred.parent_user // }; - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; let target_account = if query.user.as_ref().is_some_and(|v| v != &cred.access_key) { if !iam_store @@ -552,7 +560,9 @@ impl Operation for DeleteServiceAccount { return Err(s3_error!(InvalidArgument, "access key is empty")); } - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; let svc_account = match iam_store.get_service_account(&query.access_key).await { Ok((res, _)) => Some(res), diff --git a/rustfs/src/admin/handlers/sts.rs b/rustfs/src/admin/handlers/sts.rs index 327c81d8..5d487df4 100644 --- a/rustfs/src/admin/handlers/sts.rs +++ b/rustfs/src/admin/handlers/sts.rs @@ -16,11 +16,11 @@ use crate::{ admin::router::Operation, auth::{check_key_valid, get_session_token}, }; -use ecstore::bucket::utils::serialize; use http::StatusCode; -use iam::{manager::get_token_signing_key, sys::SESSION_POLICY_NAME}; use matchit::Params; -use policy::{auth::get_new_credentials_with_metadata, policy::Policy}; +use rustfs_ecstore::bucket::utils::serialize; +use rustfs_iam::{manager::get_token_signing_key, sys::SESSION_POLICY_NAME}; +use rustfs_policy::{auth::get_new_credentials_with_metadata, policy::Policy}; use rustfs_utils::crypto::base64_encode; use s3s::{ Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, @@ -112,7 +112,9 @@ impl Operation for AssumeRoleHandle { // warn!("AssumeRole get cred {:?}", &user); // warn!("AssumeRole get body {:?}", &body); - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; if let Err(_err) = iam_store.policy_db_get(&cred.access_key, &cred.groups).await { return Err(s3_error!(InvalidArgument, "invalid policy arg")); diff --git a/rustfs/src/admin/handlers/tier.rs b/rustfs/src/admin/handlers/tier.rs index 569aebcd..ff1efa10 100644 --- a/rustfs/src/admin/handlers/tier.rs +++ b/rustfs/src/admin/handlers/tier.rs @@ -14,11 +14,11 @@ // limitations under the License. use http::{HeaderMap, StatusCode}; -use time::OffsetDateTime; //use iam::get_global_action_cred; use matchit::Params; use s3s::{Body, S3Error, S3ErrorCode, S3Request, S3Response, S3Result, header::CONTENT_TYPE, s3_error}; use serde_urlencoded::from_bytes; +use time::OffsetDateTime; use tracing::{debug, warn}; use crate::{ @@ -26,7 +26,7 @@ use crate::{ auth::{check_key_valid, get_session_token}, }; -use ecstore::{ +use rustfs_ecstore::{ config::storageclass, global::GLOBAL_TierConfigMgr, tier::{ diff --git a/rustfs/src/admin/handlers/trace.rs b/rustfs/src/admin/handlers/trace.rs index af580f89..843f6e0d 100644 --- a/rustfs/src/admin/handlers/trace.rs +++ b/rustfs/src/admin/handlers/trace.rs @@ -12,11 +12,11 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::{GLOBAL_Endpoints, rpc::PeerRestClient}; use http::StatusCode; use hyper::Uri; -use madmin::service_commands::ServiceTraceOpts; use matchit::Params; +use rustfs_ecstore::{GLOBAL_Endpoints, rpc::PeerRestClient}; +use rustfs_madmin::service_commands::ServiceTraceOpts; use s3s::{Body, S3Request, S3Response, S3Result, s3_error}; use tracing::warn; diff --git a/rustfs/src/admin/handlers/user.rs b/rustfs/src/admin/handlers/user.rs index 1b981529..64ebcb41 100644 --- a/rustfs/src/admin/handlers/user.rs +++ b/rustfs/src/admin/handlers/user.rs @@ -16,18 +16,18 @@ use crate::{ admin::{router::Operation, utils::has_space_be}, auth::{check_key_valid, get_condition_values, get_session_token}, }; -use ecstore::global::get_global_action_cred; use http::{HeaderMap, StatusCode}; -use iam::{ +use matchit::Params; +use rustfs_ecstore::global::get_global_action_cred; +use rustfs_iam::{ store::{GroupInfo, MappedPolicy, UserType}, sys::NewServiceAccountOpts, }; -use madmin::{ +use rustfs_madmin::{ AccountStatus, AddOrUpdateUserReq, IAMEntities, IAMErrEntities, IAMErrEntity, IAMErrPolicyEntity, user::{ImportIAMResult, SRSessionPolicy, SRSvcAccCreate}, }; -use matchit::Params; -use policy::policy::{ +use rustfs_policy::policy::{ Args, action::{Action, AdminAction}, }; @@ -104,7 +104,9 @@ impl Operation for AddUser { } } - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; if let Some(user) = iam_store.get_user(ak).await { if (user.credentials.is_temp() || user.credentials.is_service_account()) && cred.parent_user == ak { @@ -180,7 +182,9 @@ impl Operation for SetUserStatus { let status = AccountStatus::try_from(query.status.as_deref().unwrap_or_default()) .map_err(|e| S3Error::with_message(S3ErrorCode::InvalidArgument, e))?; - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; iam_store .set_user_status(ak, status) @@ -213,7 +217,9 @@ impl Operation for ListUsers { } }; - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; let users = { if !query.bucket.is_empty() { @@ -266,7 +272,9 @@ impl Operation for RemoveUser { return Err(s3_error!(InvalidArgument, "access key is empty")); } - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; let (is_temp, _) = iam_store .is_temp_user(ak) @@ -327,7 +335,9 @@ impl Operation for GetUserInfo { return Err(s3_error!(InvalidArgument, "access key is empty")); } - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; let Some(input_cred) = req.credentials else { return Err(s3_error!(InvalidRequest, "get cred failed")); @@ -401,7 +411,9 @@ impl Operation for ExportIam { let (_cred, _owner) = check_key_valid(get_session_token(&req.uri, &req.headers).unwrap_or_default(), &input_cred.access_key).await?; - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; let mut zip_writer = ZipWriter::new(Cursor::new(Vec::new())); let options = SimpleFileOptions::default(); @@ -410,7 +422,7 @@ impl Operation for ExportIam { let file_path = path_join_buf(&[IAM_ASSETS_DIR, file]); match file { ALL_POLICIES_FILE => { - let policies: HashMap = iam_store + let policies: HashMap = iam_store .list_polices("") .await .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; @@ -615,7 +627,9 @@ impl Operation for ImportIam { let mut zip_reader = ZipArchive::new(Cursor::new(body)).map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; - let Ok(iam_store) = iam::get() else { return Err(s3_error!(InvalidRequest, "iam not init")) }; + let Ok(iam_store) = rustfs_iam::get() else { + return Err(s3_error!(InvalidRequest, "iam not init")); + }; let skipped = IAMEntities::default(); let mut removed = IAMEntities::default(); @@ -637,7 +651,7 @@ impl Operation for ImportIam { }; if let Some(file_content) = file_content { - let policies: HashMap = serde_json::from_slice(&file_content) + let policies: HashMap = serde_json::from_slice(&file_content) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; for (name, policy) in policies { if policy.id.is_empty() { @@ -723,7 +737,7 @@ impl Operation for ImportIam { .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; for (group_name, group_info) in groups { if let Err(e) = iam_store.get_group_description(&group_name).await { - if matches!(e, iam::error::Error::NoSuchGroup(_)) || has_space_be(&group_name) { + if matches!(e, rustfs_iam::error::Error::NoSuchGroup(_)) || has_space_be(&group_name) { return Err(s3_error!(InvalidArgument, "group not found or has space be")); } } @@ -763,7 +777,7 @@ impl Operation for ImportIam { } let sp = if let Some(ps) = req.session_policy.as_str() { - let sp = policy::policy::Policy::parse_config(ps.as_bytes()) + let sp = rustfs_policy::policy::Policy::parse_config(ps.as_bytes()) .map_err(|e| S3Error::with_message(S3ErrorCode::InternalError, e.to_string()))?; Some(sp) } else { @@ -777,7 +791,7 @@ impl Operation for ImportIam { let mut update = true; if let Err(e) = iam_store.get_service_account(&req.access_key).await { - if !matches!(e, iam::error::Error::NoSuchServiceAccount(_)) { + if !matches!(e, rustfs_iam::error::Error::NoSuchServiceAccount(_)) { return Err(s3_error!(InvalidArgument, "failed to get service account {ak} {e}")); } update = false; @@ -838,7 +852,7 @@ impl Operation for ImportIam { let has_temp = match iam_store.is_temp_user(&user_name).await { Ok((has_temp, _)) => has_temp, Err(e) => { - if !matches!(e, iam::error::Error::NoSuchUser(_)) { + if !matches!(e, rustfs_iam::error::Error::NoSuchUser(_)) { return Err(s3_error!(InternalError, "is temp user failed, name: {user_name}, err: {e}")); } false @@ -934,7 +948,7 @@ impl Operation for ImportIam { let has_temp = match iam_store.is_temp_user(&user_name).await { Ok((has_temp, _)) => has_temp, Err(e) => { - if !matches!(e, iam::error::Error::NoSuchUser(_)) { + if !matches!(e, rustfs_iam::error::Error::NoSuchUser(_)) { return Err(s3_error!(InternalError, "is temp user failed, name: {user_name}, err: {e}")); } false diff --git a/rustfs/src/admin/router.rs b/rustfs/src/admin/router.rs index e0b01dc2..ee426c60 100644 --- a/rustfs/src/admin/router.rs +++ b/rustfs/src/admin/router.rs @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::rpc::verify_rpc_signature; use hyper::HeaderMap; use hyper::Method; use hyper::StatusCode; @@ -20,6 +19,7 @@ use hyper::Uri; use hyper::http::Extensions; use matchit::Params; use matchit::Router; +use rustfs_ecstore::rpc::verify_rpc_signature; use s3s::Body; use s3s::S3Request; use s3s::S3Response; diff --git a/rustfs/src/admin/rpc.rs b/rustfs/src/admin/rpc.rs index 45048bc0..bc03cae5 100644 --- a/rustfs/src/admin/rpc.rs +++ b/rustfs/src/admin/rpc.rs @@ -15,14 +15,14 @@ use super::router::AdminOperation; use super::router::Operation; use super::router::S3Router; -use ecstore::disk::DiskAPI; -use ecstore::disk::WalkDirOptions; -use ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; -use ecstore::store::find_local_disk; use futures::StreamExt; use http::StatusCode; use hyper::Method; use matchit::Params; +use rustfs_ecstore::disk::DiskAPI; +use rustfs_ecstore::disk::WalkDirOptions; +use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; +use rustfs_ecstore::store::find_local_disk; use rustfs_utils::net::bytes_stream; use s3s::Body; use s3s::S3Request; diff --git a/rustfs/src/auth.rs b/rustfs/src/auth.rs index 5cdad22e..53367e0a 100644 --- a/rustfs/src/auth.rs +++ b/rustfs/src/auth.rs @@ -12,13 +12,13 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::global::get_global_action_cred; use http::HeaderMap; use http::Uri; -use iam::error::Error as IamError; -use iam::sys::SESSION_POLICY_NAME; -use policy::auth; -use policy::auth::get_claims_from_token_with_secret; +use rustfs_ecstore::global::get_global_action_cred; +use rustfs_iam::error::Error as IamError; +use rustfs_iam::sys::SESSION_POLICY_NAME; +use rustfs_policy::auth; +use rustfs_policy::auth::get_claims_from_token_with_secret; use s3s::S3Error; use s3s::S3ErrorCode; use s3s::S3Result; @@ -51,7 +51,7 @@ impl S3Auth for IAMAuth { return Ok(key); } - if let Ok(iam_store) = iam::get() { + if let Ok(iam_store) = rustfs_iam::get() { if let Some(id) = iam_store.get_user(access_key).await { return Ok(SecretKey::from(id.credentials.secret_key.clone())); } @@ -73,7 +73,7 @@ pub async fn check_key_valid(session_token: &str, access_key: &str) -> S3Result< let sys_cred = cred.clone(); if cred.access_key != access_key { - let Ok(iam_store) = iam::get() else { + let Ok(iam_store) = rustfs_iam::get() else { return Err(S3Error::with_message( S3ErrorCode::InternalError, format!("check_key_valid {:?}", IamError::IamSysNotInitialized), diff --git a/rustfs/src/error.rs b/rustfs/src/error.rs index b18c8563..10e3940d 100644 --- a/rustfs/src/error.rs +++ b/rustfs/src/error.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::error::StorageError; +use rustfs_ecstore::error::StorageError; use s3s::{S3Error, S3ErrorCode}; #[derive(Debug)] @@ -102,8 +102,8 @@ impl From for ApiError { } } -impl From for ApiError { - fn from(err: iam::error::Error) -> Self { +impl From for ApiError { + fn from(err: rustfs_iam::error::Error) -> Self { let serr: StorageError = err.into(); serr.into() } @@ -228,7 +228,7 @@ mod tests { #[test] fn test_api_error_from_iam_error() { - let iam_error = iam::error::Error::other("IAM test error"); + let iam_error = rustfs_iam::error::Error::other("IAM test error"); let api_error: ApiError = iam_error.into(); // IAM error is first converted to StorageError, then to ApiError diff --git a/rustfs/src/event.rs b/rustfs/src/event.rs index 90310510..981b86cd 100644 --- a/rustfs/src/event.rs +++ b/rustfs/src/event.rs @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::config::GLOBAL_ServerConfig; use rustfs_config::DEFAULT_DELIMITER; +use rustfs_ecstore::config::GLOBAL_ServerConfig; use tracing::{error, info, instrument}; #[instrument] diff --git a/rustfs/src/license.rs b/rustfs/src/license.rs index d47329f6..fe9b5422 100644 --- a/rustfs/src/license.rs +++ b/rustfs/src/license.rs @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -use appauth::token::Token; +use rustfs_appauth::token::Token; use std::io::{Error, Result}; use std::sync::OnceLock; use std::time::SystemTime; @@ -31,7 +31,7 @@ pub fn init_license(license: Option) { return; } let license = license.unwrap(); - let token = appauth::token::parse_license(&license).unwrap_or_default(); + let token = rustfs_appauth::token::parse_license(&license).unwrap_or_default(); LICENSE.set(token).unwrap_or_else(|_| { error!("Failed to set license"); diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index e8ac2668..840f8a58 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -34,26 +34,6 @@ use crate::server::{SHUTDOWN_TIMEOUT, ServiceState, ServiceStateManager, Shutdow use bytes::Bytes; use chrono::Datelike; use clap::Parser; -use common::{ - // error::{Error, Result}, - globals::set_global_addr, -}; -use ecstore::StorageAPI; -use ecstore::bucket::metadata_sys::init_bucket_metadata_sys; -use ecstore::cmd::bucket_replication::init_bucket_replication_pool; -use ecstore::config as ecconfig; -use ecstore::config::GLOBAL_ConfigSys; -use ecstore::heal::background_heal_ops::init_auto_heal; -use ecstore::rpc::make_server; -use ecstore::store_api::BucketOptions; -use ecstore::{ - endpoints::EndpointServerPools, - heal::data_scanner::init_data_scanner, - set_global_endpoints, - store::{ECStore, init_local_disks}, - update_erasure_type, -}; -use ecstore::{global::set_global_rustfs_port, notification_sys::new_global_notification_sys}; use http::{HeaderMap, Request as HttpRequest, Response}; use hyper_util::server::graceful::GracefulShutdown; use hyper_util::{ @@ -61,11 +41,28 @@ use hyper_util::{ server::conn::auto::Builder as ConnBuilder, service::TowerToHyperService, }; -use iam::init_iam_sys; use license::init_license; -use protos::proto_gen::node_service::node_service_server::NodeServiceServer; +use rustfs_common::globals::set_global_addr; use rustfs_config::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY, RUSTFS_TLS_CERT, RUSTFS_TLS_KEY}; +use rustfs_ecstore::StorageAPI; +use rustfs_ecstore::bucket::metadata_sys::init_bucket_metadata_sys; +use rustfs_ecstore::cmd::bucket_replication::init_bucket_replication_pool; +use rustfs_ecstore::config as ecconfig; +use rustfs_ecstore::config::GLOBAL_ConfigSys; +use rustfs_ecstore::heal::background_heal_ops::init_auto_heal; +use rustfs_ecstore::rpc::make_server; +use rustfs_ecstore::store_api::BucketOptions; +use rustfs_ecstore::{ + endpoints::EndpointServerPools, + heal::data_scanner::init_data_scanner, + set_global_endpoints, + store::{ECStore, init_local_disks}, + update_erasure_type, +}; +use rustfs_ecstore::{global::set_global_rustfs_port, notification_sys::new_global_notification_sys}; +use rustfs_iam::init_iam_sys; use rustfs_obs::{SystemObserver, init_obs, set_global_guard}; +use rustfs_protos::proto_gen::node_service::node_service_server::NodeServiceServer; use rustfs_utils::net::parse_and_resolve_address; use rustls::ServerConfig; use s3s::{host::MultiDomain, service::S3ServiceBuilder}; @@ -142,7 +139,7 @@ async fn run(opt: config::Opt) -> Result<()> { debug!("server_address {}", &server_address); // Set up AK and SK - ecstore::global::init_global_action_cred(Some(opt.access_key.clone()), Some(opt.secret_key.clone())); + rustfs_ecstore::global::init_global_action_cred(Some(opt.access_key.clone()), Some(opt.secret_key.clone())); set_global_rustfs_port(server_port); diff --git a/rustfs/src/storage/access.rs b/rustfs/src/storage/access.rs index a9e155f9..c2497a89 100644 --- a/rustfs/src/storage/access.rs +++ b/rustfs/src/storage/access.rs @@ -15,11 +15,11 @@ use super::ecfs::FS; use crate::auth::{check_key_valid, get_condition_values, get_session_token}; use crate::license::license_check; -use ecstore::bucket::policy_sys::PolicySys; -use iam::error::Error as IamError; -use policy::auth; -use policy::policy::action::{Action, S3Action}; -use policy::policy::{Args, BucketPolicyArgs}; +use rustfs_ecstore::bucket::policy_sys::PolicySys; +use rustfs_iam::error::Error as IamError; +use rustfs_policy::auth; +use rustfs_policy::policy::action::{Action, S3Action}; +use rustfs_policy::policy::{Args, BucketPolicyArgs}; use s3s::access::{S3Access, S3AccessContext}; use s3s::{S3Error, S3ErrorCode, S3Request, S3Result, dto::*, s3_error}; use std::collections::HashMap; @@ -39,7 +39,7 @@ pub async fn authorize_request(req: &mut S3Request, action: Action) -> S3R let req_info = req.extensions.get_mut::().expect("ReqInfo not found"); if let Some(cred) = &req_info.cred { - let Ok(iam_store) = iam::get() else { + let Ok(iam_store) = rustfs_iam::get() else { return Err(S3Error::with_message( S3ErrorCode::InternalError, format!("authorize_request {:?}", IamError::IamSysNotInitialized), diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index a9ee25ad..f33dd426 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -22,71 +22,71 @@ use crate::storage::access::ReqInfo; use crate::storage::options::copy_dst_opts; use crate::storage::options::copy_src_opts; use crate::storage::options::{extract_metadata_from_mime, get_opts}; -use api::object_store::bytes_stream; -use api::query::Context; -use api::query::Query; -use api::server::dbms::DatabaseManagerSystem; use bytes::Bytes; use chrono::DateTime; use chrono::Utc; use datafusion::arrow::csv::WriterBuilder as CsvWriterBuilder; use datafusion::arrow::json::WriterBuilder as JsonWriterBuilder; use datafusion::arrow::json::writer::JsonArray; +use rustfs_s3select_api::object_store::bytes_stream; +use rustfs_s3select_api::query::Context; +use rustfs_s3select_api::query::Query; +use rustfs_s3select_api::server::dbms::DatabaseManagerSystem; -// use ecstore::store_api::RESERVED_METADATA_PREFIX; -use ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier; -use ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; -use ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG; -use ecstore::bucket::metadata::BUCKET_POLICY_CONFIG; -use ecstore::bucket::metadata::BUCKET_REPLICATION_CONFIG; -use ecstore::bucket::metadata::BUCKET_SSECONFIG; -use ecstore::bucket::metadata::BUCKET_TAGGING_CONFIG; -use ecstore::bucket::metadata::BUCKET_VERSIONING_CONFIG; -use ecstore::bucket::metadata::OBJECT_LOCK_CONFIG; -use ecstore::bucket::metadata_sys; -use ecstore::bucket::policy_sys::PolicySys; -use ecstore::bucket::tagging::decode_tags; -use ecstore::bucket::tagging::encode_tags; -use ecstore::bucket::utils::serialize; -use ecstore::bucket::versioning_sys::BucketVersioningSys; -use ecstore::cmd::bucket_replication::ReplicationStatusType; -use ecstore::cmd::bucket_replication::ReplicationType; -use ecstore::cmd::bucket_replication::get_must_replicate_options; -use ecstore::cmd::bucket_replication::must_replicate; -use ecstore::cmd::bucket_replication::schedule_replication; -use ecstore::compress::MIN_COMPRESSIBLE_SIZE; -use ecstore::compress::is_compressible; -use ecstore::error::StorageError; -use ecstore::new_object_layer_fn; -use ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; -use ecstore::store_api::BucketOptions; -use ecstore::store_api::CompletePart; -use ecstore::store_api::DeleteBucketOptions; -use ecstore::store_api::HTTPRangeSpec; -use ecstore::store_api::MakeBucketOptions; -use ecstore::store_api::MultipartUploadResult; -use ecstore::store_api::ObjectIO; -use ecstore::store_api::ObjectOptions; -use ecstore::store_api::ObjectToDelete; -use ecstore::store_api::PutObjReader; -use ecstore::store_api::StorageAPI; +// use rustfs_ecstore::store_api::RESERVED_METADATA_PREFIX; use futures::StreamExt; use http::HeaderMap; use lazy_static::lazy_static; -use policy::auth; -use policy::policy::BucketPolicy; -use policy::policy::BucketPolicyArgs; -use policy::policy::Validator; -use policy::policy::action::Action; -use policy::policy::action::S3Action; -use query::instance::make_rustfsms; +use rustfs_ecstore::bucket::lifecycle::bucket_lifecycle_ops::validate_transition_tier; +use rustfs_ecstore::bucket::lifecycle::lifecycle::Lifecycle; +use rustfs_ecstore::bucket::metadata::BUCKET_LIFECYCLE_CONFIG; +use rustfs_ecstore::bucket::metadata::BUCKET_NOTIFICATION_CONFIG; +use rustfs_ecstore::bucket::metadata::BUCKET_POLICY_CONFIG; +use rustfs_ecstore::bucket::metadata::BUCKET_REPLICATION_CONFIG; +use rustfs_ecstore::bucket::metadata::BUCKET_SSECONFIG; +use rustfs_ecstore::bucket::metadata::BUCKET_TAGGING_CONFIG; +use rustfs_ecstore::bucket::metadata::BUCKET_VERSIONING_CONFIG; +use rustfs_ecstore::bucket::metadata::OBJECT_LOCK_CONFIG; +use rustfs_ecstore::bucket::metadata_sys; +use rustfs_ecstore::bucket::policy_sys::PolicySys; +use rustfs_ecstore::bucket::tagging::decode_tags; +use rustfs_ecstore::bucket::tagging::encode_tags; +use rustfs_ecstore::bucket::utils::serialize; +use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys; +use rustfs_ecstore::cmd::bucket_replication::ReplicationStatusType; +use rustfs_ecstore::cmd::bucket_replication::ReplicationType; +use rustfs_ecstore::cmd::bucket_replication::get_must_replicate_options; +use rustfs_ecstore::cmd::bucket_replication::must_replicate; +use rustfs_ecstore::cmd::bucket_replication::schedule_replication; +use rustfs_ecstore::compress::MIN_COMPRESSIBLE_SIZE; +use rustfs_ecstore::compress::is_compressible; +use rustfs_ecstore::error::StorageError; +use rustfs_ecstore::new_object_layer_fn; +use rustfs_ecstore::set_disk::DEFAULT_READ_BUFFER_SIZE; +use rustfs_ecstore::store_api::BucketOptions; +use rustfs_ecstore::store_api::CompletePart; +use rustfs_ecstore::store_api::DeleteBucketOptions; +use rustfs_ecstore::store_api::HTTPRangeSpec; +use rustfs_ecstore::store_api::MakeBucketOptions; +use rustfs_ecstore::store_api::MultipartUploadResult; +use rustfs_ecstore::store_api::ObjectIO; +use rustfs_ecstore::store_api::ObjectOptions; +use rustfs_ecstore::store_api::ObjectToDelete; +use rustfs_ecstore::store_api::PutObjReader; +use rustfs_ecstore::store_api::StorageAPI; use rustfs_filemeta::headers::RESERVED_METADATA_PREFIX_LOWER; use rustfs_filemeta::headers::{AMZ_DECODED_CONTENT_LENGTH, AMZ_OBJECT_TAGGING}; +use rustfs_notify::EventName; +use rustfs_policy::auth; +use rustfs_policy::policy::action::Action; +use rustfs_policy::policy::action::S3Action; +use rustfs_policy::policy::{BucketPolicy, BucketPolicyArgs, Validator}; use rustfs_rio::CompressReader; use rustfs_rio::EtagReader; use rustfs_rio::HashReader; use rustfs_rio::Reader; use rustfs_rio::WarpReader; +use rustfs_s3select_query::instance::make_rustfsms; use rustfs_utils::CompressionAlgorithm; use rustfs_utils::path::path_join_buf; use rustfs_zip::CompressionFormat; @@ -115,9 +115,6 @@ use tracing::info; use tracing::warn; use uuid::Uuid; -use ecstore::bucket::lifecycle::lifecycle::Lifecycle; -use rustfs_notify::EventName; - macro_rules! try_ { ($result:expr) => { match $result { @@ -329,7 +326,7 @@ impl S3 for FS { let event_args = rustfs_notify::event::EventArgs { event_name: EventName::BucketCreated, bucket_name: bucket.clone(), - object: ecstore::store_api::ObjectInfo { ..Default::default() }, + object: rustfs_ecstore::store_api::ObjectInfo { ..Default::default() }, req_params: rustfs_utils::extract_req_params_header(&req.headers), resp_elements: rustfs_utils::extract_resp_elements(&S3Response::new(output.clone())), version_id: String::new(), @@ -514,7 +511,7 @@ impl S3 for FS { let event_args = rustfs_notify::event::EventArgs { event_name: EventName::BucketRemoved, bucket_name: input.bucket, - object: ecstore::store_api::ObjectInfo { ..Default::default() }, + object: rustfs_ecstore::store_api::ObjectInfo { ..Default::default() }, req_params: rustfs_utils::extract_req_params_header(&req.headers), resp_elements: rustfs_utils::extract_resp_elements(&S3Response::new(DeleteBucketOutput {})), version_id: String::new(), @@ -585,7 +582,7 @@ impl S3 for FS { let event_args = rustfs_notify::event::EventArgs { event_name: EventName::ObjectRemovedDelete, bucket_name: bucket.clone(), - object: ecstore::store_api::ObjectInfo { + object: rustfs_ecstore::store_api::ObjectInfo { name: key, bucket, ..Default::default() @@ -671,7 +668,7 @@ impl S3 for FS { let event_args = rustfs_notify::event::EventArgs { event_name, bucket_name: bucket.clone(), - object: ecstore::store_api::ObjectInfo { + object: rustfs_ecstore::store_api::ObjectInfo { name: dobj.object_name, bucket: bucket.clone(), ..Default::default() @@ -1376,7 +1373,7 @@ impl S3 for FS { let event_args = rustfs_notify::event::EventArgs { event_name: EventName::ObjectCreatedCompleteMultipartUpload, bucket_name: bucket_name.clone(), - object: ecstore::store_api::ObjectInfo { + object: rustfs_ecstore::store_api::ObjectInfo { name: object_name, bucket: bucket_name, ..Default::default() @@ -1668,7 +1665,7 @@ impl S3 for FS { let event_args = rustfs_notify::event::EventArgs { event_name: EventName::ObjectCreatedPutTagging, bucket_name: bucket.clone(), - object: ecstore::store_api::ObjectInfo { + object: rustfs_ecstore::store_api::ObjectInfo { name: object.clone(), bucket, ..Default::default() @@ -1735,7 +1732,7 @@ impl S3 for FS { let event_args = rustfs_notify::event::EventArgs { event_name: EventName::ObjectCreatedDeleteTagging, bucket_name: bucket.clone(), - object: ecstore::store_api::ObjectInfo { + object: rustfs_ecstore::store_api::ObjectInfo { name: object.clone(), bucket, ..Default::default() @@ -2485,7 +2482,7 @@ impl S3 for FS { let event_args = rustfs_notify::event::EventArgs { event_name: EventName::ObjectAccessedAttributes, bucket_name: bucket.clone(), - object: ecstore::store_api::ObjectInfo { + object: rustfs_ecstore::store_api::ObjectInfo { name: key.clone(), bucket, ..Default::default() diff --git a/rustfs/src/storage/error.rs b/rustfs/src/storage/error.rs index 2416550b..7ff71e87 100644 --- a/rustfs/src/storage/error.rs +++ b/rustfs/src/storage/error.rs @@ -12,9 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. -use common::error::Error; use ecstore::error::StorageError; -use s3s::{S3Error, S3ErrorCode, s3_error}; +use rustfs_common::error::Error; +use s3s::{s3_error, S3Error, S3ErrorCode}; pub fn to_s3_error(err: Error) -> S3Error { if let Some(storage_err) = err.downcast_ref::() { return match storage_err { diff --git a/rustfs/src/storage/options.rs b/rustfs/src/storage/options.rs index 755b2c7b..7a9489b1 100644 --- a/rustfs/src/storage/options.rs +++ b/rustfs/src/storage/options.rs @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -use ecstore::bucket::versioning_sys::BucketVersioningSys; -use ecstore::error::Result; -use ecstore::error::StorageError; -use ecstore::store_api::ObjectOptions; use http::{HeaderMap, HeaderValue}; use lazy_static::lazy_static; +use rustfs_ecstore::bucket::versioning_sys::BucketVersioningSys; +use rustfs_ecstore::error::Result; +use rustfs_ecstore::error::StorageError; +use rustfs_ecstore::store_api::ObjectOptions; use rustfs_utils::path::is_dir_object; use std::collections::HashMap; use uuid::Uuid;