diff --git a/Cargo.lock b/Cargo.lock index 39bede6a..c52b23e9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -28,6 +28,102 @@ dependencies = [ "version_check", ] +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anstream" +version = "0.6.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418c75fa768af9c03be99d17643f93f79bbba589895012a80e3452a19ddda15b" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "is_terminal_polyfill", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "038dfcf04a5feb68e9c60b21c9625a54c2c0616e79b72b0fd87075a056ae1d1b" + +[[package]] +name = "anstyle-parse" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c03a11a9034d92058ceb6ee011ce58af4a9bf61491aa7e1e59ecd24bd40d22d4" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad186efb764318d35165f1758e7dcef3b10628e26d41a44bc5550652e6804391" +dependencies = [ + "windows-sys 0.52.0", +] + +[[package]] +name = "anstyle-wincon" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61a38449feb7068f52bb06c12759005cf459ee52bb4adc1d5a7c4322d716fb19" +dependencies = [ + "anstyle", + "windows-sys 0.52.0", +] + +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + +[[package]] +name = "arrayvec" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" + +[[package]] +name = "async-trait" +version = "0.1.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6fa2087f2753a7da8cc1c0dbfcf89579dd57458e36769de5ac750b4671737ca" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + +[[package]] +name = "atomic-waker" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" + [[package]] name = "autocfg" version = "1.3.0" @@ -49,18 +145,52 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "339abbe78e73178762e23bea9dfd08e697eb3f3301cd4be981c0f78ba5859195" +dependencies = [ + "outref", + "vsimd", +] + [[package]] name = "bitflags" version = "1.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" +[[package]] +name = "bitflags" +version = "2.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" + +[[package]] +name = "block-buffer" +version = "0.10.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3078c7629b62d3f0439517fa394996acacc5cbc91c5a20d8c658e77abd503a71" +dependencies = [ + "generic-array", +] + [[package]] name = "bytes" version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "514de17de45fdb8dc022b1a7975556c53c86f9f0aa5f534b98977b171857c2c9" +[[package]] +name = "bytestring" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d80203ea6b29df88012294f62733de21cfeab47f17b41af3a38bc30a03ee72" +dependencies = [ + "bytes", +] + [[package]] name = "cc" version = "1.0.100" @@ -73,6 +203,89 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.38" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21f936df1771bf62b77f047b726c4625ff2e8aa607c01ec06e5a05bd8463401" +dependencies = [ + "num-traits", +] + +[[package]] +name = "clap" +version = "4.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5db83dced34638ad474f39f250d7fea9598bdd239eaced1bdf45d597da0f433f" +dependencies = [ + "clap_builder", + "clap_derive", +] + +[[package]] +name = "clap_builder" +version = "4.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e204572485eb3fbf28f871612191521df159bc3e15a9f5064c66dba3a8c05f" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c780290ccf4fb26629baa7a1081e68ced113f1d3ec302fa5948f1c381ebf06c6" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "clap_lex" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b82cf0babdbd58558212896d1a4272303a57bdb245c2bf1147185fb45640e70" + +[[package]] +name = "colorchoice" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b6a852b24ab71dffc585bcb46eaf7959d175cb865a7152e35b348d1b2960422" + +[[package]] +name = "cpufeatures" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53fe5e26ff1b7aef8bca9c6080520cfb8d9333c7568e1829cef191a9723e5504" +dependencies = [ + "libc", +] + +[[package]] +name = "crc32fast" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crypto-common" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +dependencies = [ + "generic-array", + "typenum", +] + [[package]] name = "deranged" version = "0.3.11" @@ -82,6 +295,51 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "digest" +version = "0.10.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" +dependencies = [ + "block-buffer", + "crypto-common", + "subtle", +] + +[[package]] +name = "ecstore" +version = "0.1.0" +dependencies = [ + "anyhow", + "async-trait", + "bytes", + "futures", + "lazy_static", + "netif", + "reed-solomon-erasure", + "regex", + "s3s", + "serde", + "thiserror", + "tokio", + "tracing", + "transform-stream", + "url", + "uuid", +] + +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "form_urlencoded" version = "1.2.1" @@ -180,6 +438,16 @@ dependencies = [ "slab", ] +[[package]] +name = "generic-array" +version = "0.14.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +dependencies = [ + "typenum", + "version_check", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -197,6 +465,25 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "h2" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa82e28a107a8cc405f0839610bdc9b15f1e25ec7d696aa5cf173edbcb1486ab" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + [[package]] name = "hashbrown" version = "0.12.3" @@ -206,12 +493,124 @@ dependencies = [ "ahash", ] +[[package]] +name = "hashbrown" +version = "0.14.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" + +[[package]] +name = "heck" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" + [[package]] name = "hermit-abi" version = "0.3.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" +[[package]] +name = "hex-simd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f7685beb53fc20efc2605f32f5d51e9ba18b8ef237961d1760169d2290d3bee" +dependencies = [ + "outref", + "vsimd", +] + +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + +[[package]] +name = "http" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "21b9ddb458710bc376481b842f5da65cdf31522de232c1ca8146abce2a358258" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1cac85db508abc24a2e48553ba12a996e87244a0395ce011e62b37158745d643" +dependencies = [ + "bytes", + "http", +] + +[[package]] +name = "http-body-util" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793429d76616a256bcb62c2a2ec2bed781c8307e797e2598c50010f2bee2544f" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fcc0b4a115bf80b728eb8ea024ad5bd707b615bfed49e0665b6e0f86fd082d9" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "1.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe575dd17d0862a9a33781c8c4696a55c320909004a67a00fb286ba8b1bc496d" +dependencies = [ + "bytes", + "futures-channel", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "smallvec", + "tokio", +] + +[[package]] +name = "hyper-util" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" +dependencies = [ + "bytes", + "futures-util", + "http", + "http-body", + "hyper", + "pin-project-lite", + "tokio", +] + [[package]] name = "idna" version = "0.5.0" @@ -222,6 +621,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "2.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "168fb715dda47215e360912c096649d23d58bf392ac62f73919e831745e40f26" +dependencies = [ + "equivalent", + "hashbrown 0.14.5", +] + [[package]] name = "instant" version = "0.1.13" @@ -231,6 +640,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "is_terminal_polyfill" +version = "1.70.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" + [[package]] name = "itoa" version = "1.0.11" @@ -277,7 +692,16 @@ version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e999beba7b6e8345721bd280141ed958096a2e4abdf74f67ff4ce49b4b54e47a" dependencies = [ - "hashbrown", + "hashbrown 0.12.3", +] + +[[package]] +name = "matchers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +dependencies = [ + "regex-automata 0.1.10", ] [[package]] @@ -286,6 +710,18 @@ version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "mime" +version = "0.3.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" + +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.4" @@ -295,6 +731,37 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4a650543ca06a924e8b371db273b2756685faae30f8487da1b56505a8f78b0c" +dependencies = [ + "libc", + "wasi", + "windows-sys 0.48.0", +] + +[[package]] +name = "netif" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29a01b9f018d6b7b277fef6c79fdbd9bf17bb2d1e298238055cafab49baa5ee" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -305,12 +772,30 @@ dependencies = [ "winapi", ] +[[package]] +name = "nugine-rust-utils" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04dcd9cfa92246a9c7ca0671e00733c4e9d77ee1fa0ae08c9a181b7c8802aea2" +dependencies = [ + "simdutf8", +] + [[package]] name = "num-conv" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", +] + [[package]] name = "num_cpus" version = "1.16.0" @@ -336,6 +821,12 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "outref" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4030760ffd992bef45b0ae3f10ce1aba99e33464c90d14dd7c039884963ddc7a" + [[package]] name = "overload" version = "0.1.1" @@ -350,7 +841,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.10", ] [[package]] @@ -362,11 +863,24 @@ dependencies = [ "cfg-if", "instant", "libc", - "redox_syscall", + "redox_syscall 0.2.16", "smallvec", "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.2", + "smallvec", + "windows-targets 0.52.5", +] + [[package]] name = "percent-encoding" version = "2.3.1" @@ -400,6 +914,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "quick-xml" +version = "0.31.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quote" version = "1.0.36" @@ -415,7 +939,16 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fb5a58c1855b4b6819d59012155603f0b22ad30cad752600aadfcb695265519a" dependencies = [ - "bitflags", + "bitflags 1.3.2", +] + +[[package]] +name = "redox_syscall" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c82cf8cff14456045f55ec4241383baeff27af886adb72ffb2162f99911de0fd" +dependencies = [ + "bitflags 2.6.0", ] [[package]] @@ -426,11 +959,55 @@ checksum = "7263373d500d4d4f505d43a2a662d475a894aa94503a1ee28e9188b5f3960d4f" dependencies = [ "libm", "lru", - "parking_lot", + "parking_lot 0.11.2", "smallvec", "spin", ] +[[package]] +name = "regex" +version = "1.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata 0.4.7", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-automata" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" +dependencies = [ + "regex-syntax 0.6.29", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax 0.8.4", +] + +[[package]] +name = "regex-syntax" +version = "0.6.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" + +[[package]] +name = "regex-syntax" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -441,11 +1018,61 @@ checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" name = "rustfs" version = "0.1.0" dependencies = [ - "store", - "time", + "clap", + "ecstore", + "hyper-util", + "s3s", + "tokio", "tracing-subscriber", ] +[[package]] +name = "ryu" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" + +[[package]] +name = "s3s" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "67e6cdc8002708b435946eec39afa13c43e4288d1de6316a12816e4cfaaa6c2c" +dependencies = [ + "arrayvec", + "async-trait", + "atoi", + "base64-simd", + "bytes", + "bytestring", + "chrono", + "crc32fast", + "futures", + "hex-simd", + "hmac", + "http-body", + "http-body-util", + "httparse", + "hyper", + "itoa", + "memchr", + "mime", + "nom", + "nugine-rust-utils", + "pin-project-lite", + "quick-xml", + "serde", + "serde_urlencoded", + "sha1", + "sha2", + "smallvec", + "thiserror", + "time", + "tracing", + "transform-stream", + "urlencoding", + "zeroize", +] + [[package]] name = "scopeguard" version = "1.2.0" @@ -472,6 +1099,40 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + +[[package]] +name = "sha2" +version = "0.10.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "793db75ad2bcafc3ffa7c68b215fee268f537982cd901d132f89c6343f3a3dc8" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sharded-slab" version = "0.1.7" @@ -481,6 +1142,21 @@ dependencies = [ "lazy_static", ] +[[package]] +name = "signal-hook-registry" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9e9e0b4211b72e7b8b6e85c807d36c212bdb33ea8587f7569562a84df5465b1" +dependencies = [ + "libc", +] + +[[package]] +name = "simdutf8" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f27f6278552951f1f2b8cf9da965d10969b2efdea95a6ec47987ab46edfe263a" + [[package]] name = "slab" version = "0.4.9" @@ -496,6 +1172,16 @@ version = "1.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c5e1a9a646d36c3599cd173a41282daf47c44583ad367b8e6837255952e5c67" +[[package]] +name = "socket2" +version = "0.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce305eb0b4296696835b71df73eb912e0f1ffd2556a501fcede6e0c50349191c" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + [[package]] name = "spin" version = "0.9.8" @@ -503,18 +1189,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" [[package]] -name = "store" -version = "0.1.0" -dependencies = [ - "bytes", - "futures", - "reed-solomon-erasure", - "thiserror", - "tokio", - "transform-stream", - "url", - "uuid", -] +name = "strsim" +version = "0.11.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" + +[[package]] +name = "subtle" +version = "2.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" @@ -610,9 +1294,16 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ba4f4a02a7a80d6f274636f0aa95c7e383b912d41fe721a31f29e29698585a4a" dependencies = [ "backtrace", + "bytes", + "libc", + "mio", "num_cpus", + "parking_lot 0.12.3", "pin-project-lite", + "signal-hook-registry", + "socket2", "tokio-macros", + "windows-sys 0.48.0", ] [[package]] @@ -626,6 +1317,41 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-util" +version = "0.7.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cf6b47b3771c49ac75ad09a6162f53ad4b8088b76ac60e8ec1455b31a189fe1" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tracing" +version = "0.1.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" +dependencies = [ + "pin-project-lite", + "tracing-attributes", + "tracing-core", +] + +[[package]] +name = "tracing-attributes" +version = "0.1.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34704c8d6ebcbc939824180af020566b01a7c01f80641264eba0999f6c2b6be7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "tracing-core" version = "0.1.32" @@ -653,10 +1379,15 @@ version = "0.3.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ad0f048c97dbd9faa9b7df56362b8ebcaa52adb06b498c050d2f4e32f90a7a8b" dependencies = [ + "matchers", "nu-ansi-term", + "once_cell", + "regex", "sharded-slab", "smallvec", "thread_local", + "time", + "tracing", "tracing-core", "tracing-log", ] @@ -670,6 +1401,12 @@ dependencies = [ "futures-core", ] +[[package]] +name = "typenum" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" + [[package]] name = "unicode-bidi" version = "0.3.15" @@ -702,6 +1439,18 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "urlencoding" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "daf8dba3b7eb870caf1ddeed7bc9d2a049f3cfdfae7cb521b087cc33ae4c49da" + +[[package]] +name = "utf8parse" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" + [[package]] name = "uuid" version = "1.8.0" @@ -720,6 +1469,12 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "vsimd" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -747,3 +1502,148 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.5", +] + +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets 0.52.5", +] + +[[package]] +name = "windows-targets" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +dependencies = [ + "windows_aarch64_gnullvm 0.48.5", + "windows_aarch64_msvc 0.48.5", + "windows_i686_gnu 0.48.5", + "windows_i686_msvc 0.48.5", + "windows_x86_64_gnu 0.48.5", + "windows_x86_64_gnullvm 0.48.5", + "windows_x86_64_msvc 0.48.5", +] + +[[package]] +name = "windows-targets" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6f0713a46559409d202e70e28227288446bf7841d3211583a4b53e3f6d96e7eb" +dependencies = [ + "windows_aarch64_gnullvm 0.52.5", + "windows_aarch64_msvc 0.52.5", + "windows_i686_gnu 0.52.5", + "windows_i686_gnullvm", + "windows_i686_msvc 0.52.5", + "windows_x86_64_gnu 0.52.5", + "windows_x86_64_gnullvm 0.52.5", + "windows_x86_64_msvc 0.52.5", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7088eed71e8b8dda258ecc8bac5fb1153c5cffaf2578fc8ff5d61e23578d3263" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9985fd1504e250c615ca5f281c3f7a6da76213ebd5ccc9561496568a2752afb6" + +[[package]] +name = "windows_i686_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88ba073cf16d5372720ec942a8ccbf61626074c6d4dd2e745299726ce8b89670" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87f4261229030a858f36b459e748ae97545d6f1ec60e5e0d6a3d32e0dc232ee9" + +[[package]] +name = "windows_i686_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db3c2bf3d13d5b658be73463284eaf12830ac9a26a90c717b7f771dfe97487bf" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e4246f76bdeff09eb48875a0fd3e2af6aada79d409d33011886d3e1581517d9" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "852298e482cd67c356ddd9570386e2862b5673c85bd5f88df9ab6802b334c596" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bec47e5bfd1bff0eeaf6d8b485cc1074891a197ab4225d504cb7a1ab88b02bf0" + +[[package]] +name = "zeroize" +version = "1.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" diff --git a/Cargo.toml b/Cargo.toml index ee171f3d..efc56d59 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [workspace] resolver = "2" -members = ["rustfs", "store"] +members = ["rustfs", "ecstore"] [workspace.package] edition = "2021" @@ -18,4 +18,6 @@ http = "1.1.0" thiserror = "1.0.61" time = "0.3.36" async-trait = "0.1.80" -tokio = { version = "1.38.0", features = ["macros", "rt", "rt-multi-thread"] } +# tokio = { version = "1.38.0", features = ["macros", "rt", "rt-multi-thread", "fs", "io-util"] } +tokio = { version = "1.38.0", features = ["full"] } +tokio-util = { version = "0.7.8", features = ["io"] } diff --git a/store/Cargo.toml b/ecstore/Cargo.toml similarity index 71% rename from store/Cargo.toml rename to ecstore/Cargo.toml index 81e6fd4d..d849326b 100644 --- a/store/Cargo.toml +++ b/ecstore/Cargo.toml @@ -1,5 +1,5 @@ [package] -name = "store" +name = "ecstore" version = "0.1.0" edition.workspace = true license.workspace = true @@ -17,3 +17,11 @@ bytes.workspace = true tokio.workspace = true thiserror.workspace = true futures.workspace = true +anyhow = "1.0.86" +serde.workspace = true +lazy_static = "1.5.0" +regex = "1.10.5" +netif = "0.1.6" +async-trait = "0.1.80" +s3s = "0.10.0" +tracing.workspace = true diff --git a/ecstore/src/disks_layout.rs b/ecstore/src/disks_layout.rs new file mode 100644 index 00000000..2b1e644a --- /dev/null +++ b/ecstore/src/disks_layout.rs @@ -0,0 +1,376 @@ +use std::collections::{HashMap, HashSet}; + +use anyhow::Error; +use serde::Deserialize; + +use super::ellipses::*; + +#[derive(Deserialize, Debug, Default)] +pub struct PoolDisksLayout { + pub cmdline: String, + pub layout: Vec>, +} + +#[derive(Deserialize, Debug, Default)] +pub struct DisksLayout { + pub legacy: bool, + pub pools: Vec, +} + +impl DisksLayout { + pub fn new(args: Vec) -> Result { + if args.is_empty() { + return Err(Error::msg("Invalid argument")); + } + + let mut ok = true; + for arg in args.iter() { + ok = ok && !has_ellipses(&vec![arg.to_string()]) + } + + // TODO: from env + let set_drive_count: usize = 0; + + if ok { + let set_args = get_all_sets(set_drive_count, &args)?; + + return Ok(DisksLayout { + legacy: true, + pools: vec![PoolDisksLayout { + layout: set_args, + cmdline: args.join(" "), + }], + }); + } + + let mut ret = DisksLayout { + pools: Vec::new(), + ..Default::default() + }; + + for arg in args.iter() { + let varg = vec![arg.to_string()]; + + if !has_ellipses(&varg) && args.len() > 1 { + return Err(Error::msg("所有参数必须包含省略号以用于池扩展")); + } + + let set_args = get_all_sets(set_drive_count, &varg)?; + + ret.pools.push(PoolDisksLayout { + layout: set_args, + cmdline: arg.clone(), + }) + } + + Ok(ret) + } +} + +fn get_all_sets(set_drive_count: usize, args: &Vec) -> Result>, Error> { + let set_args; + if !has_ellipses(args) { + let set_indexes: Vec>; + if args.len() > 1 { + let totalsizes = vec![args.len()]; + set_indexes = get_set_indexes(args, &totalsizes, set_drive_count, &Vec::new())?; + } else { + set_indexes = vec![vec![args.len()]]; + } + + let mut s = EndpointSet { + endpoints: args.clone(), + set_indexes, + ..Default::default() + }; + + set_args = s.get(); + } else { + let mut s = EndpointSet::new(args, set_drive_count)?; + set_args = s.get(); + } + + let mut seen = HashSet::with_capacity(set_args.len()); + for args in set_args.iter() { + for arg in args { + if seen.contains(arg) { + return Err(Error::msg(format!( + "Input args {} has duplicate ellipses", + arg + ))); + } + seen.insert(arg); + } + } + + Ok(set_args) +} + +#[derive(Debug, Default)] +pub struct EndpointSet { + pub arg_patterns: Vec, + pub endpoints: Vec, + pub set_indexes: Vec>, +} + +impl EndpointSet { + pub fn new(args: &Vec, set_div_count: usize) -> Result { + let mut arg_patterns = Vec::with_capacity(args.len()); + for arg in args.iter() { + arg_patterns.push(find_ellipses_patterns(arg.as_str())?); + } + + let totalsizes = get_total_sizes(&arg_patterns); + let set_indexes = get_set_indexes(args, &totalsizes, set_div_count, &arg_patterns)?; + + Ok(EndpointSet { + set_indexes, + arg_patterns, + ..Default::default() + }) + } + + pub fn get(&mut self) -> Vec> { + let mut sets: Vec> = Vec::new(); + let eps = self.get_endpoints(); + + let mut start = 0; + for sidx in self.set_indexes.iter() { + for idx in sidx { + let end = idx + start; + sets.push(eps[start..end].to_vec()); + start = end; + } + } + sets + } + + fn get_endpoints(&mut self) -> Vec { + if !self.endpoints.is_empty() { + return self.endpoints.clone(); + } + + let mut endpoints = Vec::new(); + for ap in self.arg_patterns.iter() { + let aps = ap.expand(); + for bs in aps { + endpoints.push(bs.join("")); + } + } + + self.endpoints = endpoints; + + self.endpoints.clone() + } +} + +// fn parse_endpoint_set(set_div_count: usize, args: &Vec) -> Result { +// let mut arg_patterns = Vec::with_capacity(args.len()); +// for arg in args.iter() { +// arg_patterns.push(find_ellipses_patterns(arg.as_str())?); +// } + +// let totalsizes = get_total_sizes(&arg_patterns); +// let set_indexes = get_set_indexes(args, &totalsizes, set_div_count, &arg_patterns)?; + +// Ok(EndpointSet { +// set_indexes, +// arg_patterns, +// ..Default::default() +// }) +// } + +static SET_SIZES: [usize; 15] = [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + +fn gcd(mut x: usize, mut y: usize) -> usize { + while y != 0 { + let t = y; + y = x % y; + x = t; + } + x +} + +fn get_divisible_size(totalsizes: &Vec) -> usize { + let mut ret = totalsizes[0]; + for s in totalsizes.iter() { + ret = gcd(ret, *s) + } + ret +} + +fn possible_set_counts(set_size: usize) -> Vec { + let mut ss = Vec::new(); + for s in SET_SIZES { + if set_size % s == 0 { + ss.push(s); + } + } + ss +} + +fn is_valid_set_size(count: usize) -> bool { + &count >= SET_SIZES.first().unwrap() && &count <= SET_SIZES.last().unwrap() +} + +fn common_set_drive_count(divisible_size: usize, set_counts: Vec) -> usize { + // prefers set_counts to be sorted for optimal behavior. + if &divisible_size < set_counts.last().unwrap_or(&0) { + return divisible_size; + } + + let mut prev_d = divisible_size / set_counts[0]; + let mut set_size = 0; + for cnt in set_counts { + if divisible_size % cnt == 0 { + let d = divisible_size / cnt; + if d <= prev_d { + prev_d = d; + set_size = cnt; + } + } + } + set_size +} + +fn possible_set_counts_with_symmetry( + set_counts: Vec, + arg_patterns: &Vec, +) -> Vec { + let mut new_set_counts: HashMap = HashMap::new(); + + for ss in set_counts { + let mut symmetry = false; + for arg_pattern in arg_patterns { + for p in arg_pattern.inner.iter() { + if p.seq.len() > ss { + symmetry = p.seq.len() % ss == 0; + } else { + symmetry = ss % p.seq.len() == 0; + } + } + } + + if !new_set_counts.contains_key(&ss) && (symmetry || arg_patterns.is_empty()) { + new_set_counts.insert(ss, ()); + } + } + + let mut set_counts: Vec = Vec::from_iter(new_set_counts.keys().cloned()); + set_counts.sort_unstable(); + + set_counts +} + +fn get_set_indexes( + args: &Vec, + totalsizes: &Vec, + set_div_count: usize, + arg_patterns: &Vec, +) -> Result>, Error> { + if args.is_empty() || totalsizes.is_empty() { + return Err(Error::msg("Invalid argument")); + } + + for size in totalsizes.iter() { + if size.lt(&SET_SIZES[0]) || size < &set_div_count { + return Err(Error::msg(format!( + "Incorrect number of endpoints provided,size {}", + size + ))); + } + } + + let common_size = get_divisible_size(totalsizes); + let mut set_counts = possible_set_counts(common_size); + if set_counts.is_empty() { + return Err(Error::msg("Incorrect number of endpoints provided2")); + } + + let set_size; + + if set_div_count > 0 { + let mut found = false; + for ss in set_counts { + if ss == set_div_count { + found = true + } + } + + if !found { + return Err(Error::msg("Invalid set drive count.")); + } + + set_size = set_div_count + // TODO globalCustomErasureDriveCount = true + } else { + set_counts = possible_set_counts_with_symmetry(set_counts, arg_patterns); + + if set_counts.is_empty() { + return Err(Error::msg( + "No symmetric distribution detected with input endpoints provided", + )); + } + + set_size = common_set_drive_count(common_size, set_counts); + } + + if !is_valid_set_size(set_size) { + return Err(Error::msg("Incorrect number of endpoints provided3")); + } + + let mut set_indexs = Vec::with_capacity(totalsizes.len()); + + for size in totalsizes.iter() { + let mut sizes = Vec::with_capacity(size / set_size); + for _ in 0..size / set_size { + sizes.push(set_size); + } + + set_indexs.push(sizes) + } + + Ok(set_indexs) +} + +fn get_total_sizes(arg_patterns: &Vec) -> Vec { + let mut sizes = Vec::with_capacity(arg_patterns.len()); + for ap in arg_patterns { + let mut size = 1; + for p in ap.inner.iter() { + size *= p.seq.len() + } + + sizes.push(size) + } + sizes +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn test_parse_disks_layout_from_env_args() { + // let pattern = String::from("http://[2001:3984:3989::{001...002}]/disk{1...4}"); + // let pattern = String::from("/export{1...10}/disk{1...10}"); + let pattern = String::from("http://rustfs{1...2}:9000/mnt/disk{1...16}"); + + let mut args = Vec::new(); + args.push(pattern); + match DisksLayout::new(args) { + Ok(set) => { + for pool in set.pools { + println!("cmd: {:?}", pool.cmdline); + + for (i, set) in pool.layout.iter().enumerate() { + for (j, v) in set.iter().enumerate() { + println!("{:?}.{}: {:?}", i, j, v); + } + } + } + } + Err(err) => println!("{err:?}"), + } + } +} diff --git a/ecstore/src/ellipses.rs b/ecstore/src/ellipses.rs new file mode 100644 index 00000000..42f2ca0c --- /dev/null +++ b/ecstore/src/ellipses.rs @@ -0,0 +1,253 @@ +use lazy_static::*; + +use anyhow::Error; +use regex::Regex; + +lazy_static! { + static ref ELLIPSES_RE: Regex = Regex::new(r"(.*)(\{[0-9a-z]*\.\.\.[0-9a-z]*\})(.*)").unwrap(); +} + +// Ellipses constants +const OPEN_BRACES: &str = "{"; +const CLOSE_BRACES: &str = "}"; +const ELLIPSES: &str = "..."; + +#[derive(Debug, Default)] +pub struct Pattern { + pub prefix: String, + pub suffix: String, + pub seq: Vec, +} + +impl Pattern { + #[allow(dead_code)] + pub fn expand(&self) -> Vec { + let mut ret = Vec::with_capacity(self.suffix.len()); + for v in self.seq.iter() { + if !self.prefix.is_empty() && self.suffix.is_empty() { + ret.push(format!("{}{}", self.prefix, v)) + } else if self.prefix.is_empty() && !self.suffix.is_empty() { + ret.push(format!("{}{}", v, self.suffix)) + } else if self.prefix.is_empty() && self.suffix.is_empty() { + ret.push(v.to_string()) + } else { + ret.push(format!("{}{}{}", self.prefix, v, self.suffix)); + } + } + + ret + } +} + +#[derive(Debug)] +pub struct ArgPattern { + pub inner: Vec, +} + +impl ArgPattern { + #[allow(dead_code)] + pub fn new(inner: Vec) -> Self { + Self { inner } + } + + #[allow(dead_code)] + pub fn expand(&self) -> Vec> { + let mut ret = Vec::new(); + for v in self.inner.iter() { + ret.push(v.expand()); + } + + Self::arg_expander(&ret) + } + + fn arg_expander(lbs: &Vec>) -> Vec> { + let mut ret = Vec::new(); + + if lbs.len() == 1 { + let arr = lbs.get(0).unwrap(); + for bs in arr { + ret.push(vec![bs.to_string()]) + } + + return ret; + } + + let first = &lbs[0]; + let (_, other) = lbs.split_at(1); + let others = Vec::from(other); + // let other = lbs[1..lbs.len()]; + for bs in first { + let ots = Self::arg_expander(&others); + for obs in ots { + let mut v = obs; + v.push(bs.to_string()); + ret.push(v); + } + } + ret + } +} + +#[allow(dead_code)] +pub fn find_ellipses_patterns(arg: &str) -> Result { + let mut caps = match ELLIPSES_RE.captures(arg) { + Some(caps) => caps, + None => return Err(Error::msg("Invalid argument")), + }; + + if caps.len() == 0 { + return Err(Error::msg("Invalid format")); + } + + let mut pattens = Vec::new(); + + loop { + let m = match caps.get(1) { + Some(m) => m, + None => break, + }; + + let cs = match ELLIPSES_RE.captures(m.into()) { + Some(cs) => cs, + None => { + break; + } + }; + + let seq = caps + .get(2) + .map(|m| parse_ellipses_range(m.into()).unwrap_or(Vec::new())) + .unwrap(); + let suffix = caps + .get(3) + .map(|m| m.as_str().to_string()) + .unwrap_or(String::new()); + pattens.push(Pattern { + suffix, + seq, + ..Default::default() + }); + + if cs.len() > 0 { + caps = cs; + continue; + } + + break; + } + + if caps.len() > 0 { + let seq = caps + .get(2) + .map(|m| parse_ellipses_range(m.into()).unwrap_or(Vec::new())) + .unwrap(); + let suffix = caps + .get(3) + .map(|m| m.as_str().to_string()) + .unwrap_or(String::new()); + let prefix = caps + .get(1) + .map(|m| m.as_str().to_string()) + .unwrap_or(String::new()); + pattens.push(Pattern { + prefix, + suffix, + seq, + ..Default::default() + }); + } + + Ok(ArgPattern::new(pattens)) +} + +// has_ellipse return ture if has +#[allow(dead_code)] +pub fn has_ellipses(s: &Vec) -> bool { + let mut ret = true; + for v in s { + ret = + ret && (v.contains(ELLIPSES) || (v.contains(OPEN_BRACES) && v.contains(CLOSE_BRACES))); + } + + ret +} +// Parses an ellipses range pattern of following style +// `{1...64}` +// `{33...64}` +#[allow(dead_code)] +pub fn parse_ellipses_range(partten: &str) -> Result, Error> { + if !partten.contains(OPEN_BRACES) { + return Err(Error::msg("Invalid argument")); + } + if !partten.contains(OPEN_BRACES) { + return Err(Error::msg("Invalid argument")); + } + + let v: Vec<&str> = partten + .trim_start_matches(OPEN_BRACES) + .trim_end_matches(CLOSE_BRACES) + .split(ELLIPSES) + .collect(); + + if v.len() != 2 { + return Err(Error::msg("Invalid argument")); + } + + // let start = usize::from_str_radix(v[0], 16)?; + // let end = usize::from_str_radix(v[1], 16)?; + + let start = v[0].parse::()?; + let end = v[1].parse::()?; + + if start > end { + return Err(Error::msg( + "Invalid argument:range start cannot be bigger than end", + )); + } + + let mut ret: Vec = Vec::with_capacity(end + 1); + + for i in start..end + 1 { + if v[0].starts_with('0') && v[0].len() > 1 { + ret.push(format!("{:0witdth$}", i, witdth = v[0].len())); + } else { + ret.push(format!("{}", i)); + } + } + + Ok(ret) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_has_ellipses() { + assert_eq!(has_ellipses(vec!["/sdf".to_string()].as_ref()), false); + assert_eq!(has_ellipses(vec!["{1...3}".to_string()].as_ref()), true); + } + + #[test] + fn test_parse_ellipses_range() { + let s = "{1...16}"; + + match parse_ellipses_range(s) { + Ok(res) => { + println!("{:?}", res) + } + Err(err) => println!("{err:?}"), + }; + } + + #[test] + fn test_find_ellipses_patterns() { + use std::result::Result::Ok; + let pattern = "http://rustfs{1...2}:9000/mnt/disk{1...16}"; + // let pattern = "http://[2001:3984:3989::{01...f}]/disk{1...10}"; + match find_ellipses_patterns(pattern) { + Ok(caps) => println!("caps{caps:?}"), + Err(err) => println!("{err:?}"), + } + } +} diff --git a/ecstore/src/endpoint.rs b/ecstore/src/endpoint.rs new file mode 100644 index 00000000..5bec7893 --- /dev/null +++ b/ecstore/src/endpoint.rs @@ -0,0 +1,576 @@ +use std::{collections::HashMap, net::IpAddr, path::Path, usize}; + +use super::disks_layout::PoolDisksLayout; +use super::utils::{ + net::{is_local_host, split_host_port}, + string::new_string_set, +}; +use anyhow::Error; +use url::Url; + +pub const DEFAULT_PORT: u16 = 9000; + +// #[derive(Debug, Clone)] +// struct Node { +// url: url::Url, +// pools: Vec, +// is_local: bool, +// grid_host: String, +// } + +#[derive(PartialEq, Eq)] +pub enum EndpointType { + Undefiend, + PathEndpointType, + URLEndpointType, +} + +#[derive(Debug, Clone, PartialEq, Eq, Ord)] +pub struct Node { + pub url: url::Url, + pub pools: Vec, + pub is_local: bool, + pub grid_host: String, // TODO "scheme://host:port" +} + +impl PartialOrd for Node { + fn partial_cmp(&self, other: &Self) -> Option { + self.grid_host.partial_cmp(&other.grid_host) + } +} + +#[derive(Debug, Clone)] +pub struct Endpoint { + pub url: url::Url, + pub is_local: bool, + pub pool_idx: i32, + pub set_idx: i32, + pub disk_idx: i32, +} + +// 检查给定路径是否为空或根路径 +fn is_empty_path(path: &str) -> bool { + path == "" || path == "/" || path == "\\" +} + +// 检查给定字符串是否是IP地址 +fn is_host_ip(ip_str: &str) -> bool { + ip_str.parse::().is_ok() +} + +#[tokio::test] +async fn test_new_endpont() { + let arg = "./data"; + let ep = Endpoint::new(arg).unwrap(); + + println!("{:?}", ep); +} + +impl Endpoint { + fn new(arg: &str) -> Result { + if is_empty_path(arg) { + return Err(Error::msg("不支持空或根endpoint")); + } + + let url = Url::parse(arg).or_else(|e| match e { + url::ParseError::EmptyHost => Err(Error::msg("远程地址，域名不能为空")), + url::ParseError::IdnaError => Err(Error::msg("域名格式不正确")), + url::ParseError::InvalidPort => Err(Error::msg("端口格式不正确")), + url::ParseError::InvalidIpv4Address => Err(Error::msg("IP格式不正确")), + url::ParseError::InvalidIpv6Address => Err(Error::msg("IP格式不正确")), + url::ParseError::InvalidDomainCharacter => Err(Error::msg("域名字符格式不正确")), + // url::ParseError::RelativeUrlWithoutBase => todo!(), + // url::ParseError::RelativeUrlWithCannotBeABaseBase => todo!(), + // url::ParseError::SetHostOnCannotBeABaseUrl => todo!(), + url::ParseError::Overflow => Err(Error::msg("长度过长")), + _ => { + if is_host_ip(arg) { + return Err(Error::msg("无效的URL endpoint格式: 缺少 http 或 https")); + } + + let abs_arg = Path::new(arg).canonicalize()?; + + let abs = abs_arg.to_str().ok_or(Error::msg("绝对路径错误"))?; + let url = Url::from_file_path(abs).unwrap(); + Ok(url) + } + })?; + + if url.scheme() == "file" { + return Ok(Endpoint { + url: url, + is_local: true, + pool_idx: -1, + set_idx: -1, + disk_idx: -1, + }); + } + + if url.port().is_none() { + return Err(Error::msg("必须提供端口号")); + } + + if !(url.scheme() == "http" || url.scheme() == "https") { + return Err(Error::msg( + "URL endpoint格式无效: Scheme字段必须包含'http'或'https'", + )); + } + + // 检查路径 + let path = url.path(); + if is_empty_path(path) { + return Err(Error::msg("URL endpoint不支持空或根路径")); + } + + // TODO: Windows 系统上的路径处理 + #[cfg(windows)] + { + use std::env; + if env::consts::OS == "windows" { + // 处理 Windows 路径的特殊逻辑 + } + } + + Ok(Endpoint { + url: url, + is_local: false, + pool_idx: -1, + set_idx: -1, + disk_idx: -1, + }) + } + + // pub fn host_port_str(&self) -> String { + // if self.url.has_host() && self.port() > 0 { + // return format!("{}:{}", self.hostname(), self.port()); + // } else if self.url.has_host() && self.port() == 0 { + // return self.hostname().to_string(); + // } else if !self.url.has_host() && self.port() > 0 { + // return format!(":{}", self.port()); + // } else { + // return String::new(); + // } + // } + + // pub fn port(&self) -> u16 { + // self.url.port().unwrap_or(0) + // } + // pub fn hostname(&self) -> &str { + // self.url.host_str().unwrap_or("") + // } + + pub fn get_type(&self) -> EndpointType { + if self.url.scheme() == "file" { + return EndpointType::PathEndpointType; + } + + EndpointType::URLEndpointType + } + + pub fn to_string(&self) -> String { + self.url.as_str().to_string() + } + + // pub fn get_scheme(&self) -> String { + // self.url.scheme().to_string() + // } + + pub fn set_pool_index(&mut self, idx: i32) { + self.pool_idx = idx + } + + pub fn set_set_index(&mut self, idx: i32) { + self.set_idx = idx + } + + pub fn set_disk_index(&mut self, idx: i32) { + self.disk_idx = idx + } + + fn update_islocal(&mut self) -> Result<(), Error> { + if self.url.has_host() { + self.is_local = is_local_host( + self.url.host().unwrap(), + self.url.port().unwrap(), + DEFAULT_PORT, + ); + } + + Ok(()) + } + + fn grid_host(&self) -> String { + let host = self.url.host_str().unwrap_or(""); + let port = self.url.port().unwrap_or(0); + if port > 0 { + format!("{}://{}:{}", self.url.scheme(), host, port) + } else { + format!("{}://{}", self.url.scheme(), host) + } + } +} + +#[derive(Debug, Clone)] +pub struct Endpoints(Vec); + +impl Endpoints { + pub fn new() -> Self { + Self(Vec::new()) + } + + pub fn iter(&self) -> core::slice::Iter { + self.0.iter() + } + + pub fn iter_mut(&mut self) -> core::slice::IterMut { + self.0.iter_mut() + } + + pub fn slice(&self, start: usize, end: usize) -> Vec { + self.0.as_slice()[start..end].to_vec() + } + pub fn from_args(args: Vec) -> Result { + let mut ep_type = EndpointType::Undefiend; + let mut scheme = String::new(); + let mut eps = Vec::new(); + let mut uniq_args = new_string_set(); + for (i, arg) in args.iter().enumerate() { + let endpoint = Endpoint::new(arg)?; + if i == 0 { + ep_type = endpoint.get_type(); + scheme = endpoint.url.scheme().to_string(); + } else if endpoint.get_type() != ep_type { + return Err(Error::msg("不支持多种endpoints风格")); + } else if endpoint.url.scheme().to_string() != scheme { + return Err(Error::msg("不支持多种scheme")); + } + + let arg_str = endpoint.to_string(); + + if uniq_args.contains(arg_str.as_str()) { + return Err(Error::msg("发现重复 endpoints")); + } + + uniq_args.add(arg_str); + + eps.push(endpoint.clone()); + } + + Ok(Endpoints(eps)) + } +} + +#[warn(dead_code)] +pub struct PoolEndpointList(Vec); + +impl PoolEndpointList { + fn from_vec(v: Vec) -> Self { + Self(v) + } + + pub fn push(&mut self, es: Endpoints) { + self.0.push(es) + } + + // TODO: 解析域名，判断哪个是本地地址 + fn update_is_local(&mut self) -> Result<(), Error> { + for eps in self.0.iter_mut() { + for ep in eps.iter_mut() { + // TODO: + ep.update_islocal()? + } + } + + Ok(()) + } +} + +// PoolEndpoints represent endpoints in a given pool +// along with its setCount and setDriveCount. +#[derive(Debug)] +pub struct PoolEndpoints { + // indicates if endpoints are provided in non-ellipses style + pub legacy: bool, + pub set_count: usize, + pub drives_per_set: usize, + pub endpoints: Endpoints, + pub cmd_line: String, + pub platform: String, +} + +// EndpointServerPools - list of list of endpoints +#[derive(Debug)] +pub struct EndpointServerPools(Vec); + +impl EndpointServerPools { + pub fn new() -> Self { + Self(Vec::new()) + } + + pub fn first_is_local(&self) -> bool { + if self.0.is_empty() { + return false; + } + return self.0[0].endpoints.0[0].is_local; + } + + pub fn len(&self) -> usize { + self.0.len() + } + + pub fn iter(&self) -> core::slice::Iter<'_, PoolEndpoints> { + return self.0.iter(); + } + + pub fn push(&mut self, pes: PoolEndpoints) { + self.0.push(pes) + } + + pub fn add(&mut self, eps: PoolEndpoints) -> Result<(), Error> { + let mut exits = new_string_set(); + for peps in self.0.iter() { + for ep in peps.endpoints.0.iter() { + exits.add(ep.to_string()); + } + } + + for ep in eps.endpoints.0.iter() { + if exits.contains(&ep.to_string()) { + return Err(Error::msg("endpoints exists")); + } + } + + self.0.push(eps); + Ok(()) + } + + pub fn get_nodes(&self) -> Vec { + let mut node_map = HashMap::new(); + + for pool in self.iter() { + for ep in pool.endpoints.iter() { + let mut node = Node { + url: ep.url.clone(), + pools: vec![], + is_local: ep.is_local, + grid_host: ep.grid_host(), + }; + if !node.pools.contains(&ep.pool_idx) { + node.pools.push(ep.pool_idx) + } + + node_map.insert(node.grid_host.clone(), node); + } + } + + let mut nodes: Vec = node_map.into_iter().map(|(_, n)| n).collect(); + + nodes.sort_by(|a, b| a.cmp(b)); + + nodes + } +} + +#[derive(Debug)] +pub enum SetupType { + // UnknownSetupType - starts with unknown setup type. + UnknownSetupType, + + // FSSetupType - FS setup type enum. + FSSetupType, + + // ErasureSDSetupType - Erasure single drive setup enum. + ErasureSDSetupType, + + // ErasureSetupType - Erasure setup type enum. + ErasureSetupType, + + // DistErasureSetupType - Distributed Erasure setup type enum. + DistErasureSetupType, +} + +fn is_empty_layout(pools_layout: &Vec) -> bool { + if pools_layout.is_empty() { + return true; + } + let first_layout = &pools_layout[0]; + if first_layout.layout.is_empty() + || first_layout.layout[0].is_empty() + || first_layout.layout[0][0].is_empty() + { + return true; + } + false +} + +// 检查是否是单驱动器布局 +fn is_single_drive_layout(pools_layout: &Vec) -> bool { + if pools_layout.len() == 1 + && pools_layout[0].layout.len() == 1 + && pools_layout[0].layout[0].len() == 1 + { + true + } else { + false + } +} + +pub fn create_pool_endpoints( + server_addr: String, + pools: &Vec, +) -> Result<(Vec, SetupType), Error> { + if is_empty_layout(pools) { + return Err(Error::msg("empty layout")); + } + + // TODO: CheckLocalServerAddr + + if is_single_drive_layout(pools) { + let mut endpoint = Endpoint::new(pools[0].layout[0][0].as_str())?; + endpoint.update_islocal()?; + + if endpoint.get_type() != EndpointType::PathEndpointType { + return Err(Error::msg("use path style endpoint for single node setup")); + } + + endpoint.set_pool_index(0); + endpoint.set_set_index(0); + endpoint.set_disk_index(0); + + let mut endpoints = Vec::new(); + endpoints.push(endpoint); + + // TODO: checkCrossDeviceMounts + + return Ok((vec![Endpoints(endpoints)], SetupType::ErasureSDSetupType)); + } + + let mut ret = Vec::with_capacity(pools.len()); + + for (pool_idx, pool) in pools.iter().enumerate() { + let mut endpoints = Endpoints::new(); + for (set_idx, set_layout) in pool.layout.iter().enumerate() { + let mut eps = Endpoints::from_args(set_layout.to_owned())?; + // TODO: checkCrossDeviceMounts + for (disk_idx, ep) in eps.0.iter_mut().enumerate() { + ep.set_pool_index(pool_idx as i32); + ep.set_set_index(set_idx as i32); + ep.set_disk_index(disk_idx as i32); + + endpoints.0.push(ep.to_owned()); + } + } + + if endpoints.0.is_empty() { + return Err(Error::msg("invalid number of endpoints")); + } + + ret.push(endpoints); + } + + // TODO: + PoolEndpointList::from_vec(ret.clone()).update_is_local()?; + + let mut setup_type = SetupType::UnknownSetupType; + + // TODO: parse server port + let (_, server_port) = split_host_port(server_addr.as_str())?; + + let mut uniq_host = new_string_set(); + + for (_i, eps) in ret.iter_mut().enumerate() { + // TODO: 一些验证，参考原m + + for ep in eps.0.iter() { + if !ep.url.has_host() { + uniq_host.add(format!("localhost:{}", server_port)); + } else { + // uniq_host.add(ep.url.domain().) + } + } + } + + let erasure_type = uniq_host.to_slice().len() == 1; + + for eps in ret.iter() { + if eps.0[0].get_type() == EndpointType::PathEndpointType { + setup_type = SetupType::ErasureSetupType; + break; + } + + if eps.0[0].get_type() == EndpointType::URLEndpointType { + if erasure_type { + setup_type = SetupType::ErasureSetupType; + } else { + setup_type = SetupType::DistErasureSetupType; + } + + break; + } + } + + Ok((ret, setup_type)) +} + +// create_server_endpoints +pub fn create_server_endpoints( + server_addr: String, + pool_args: &Vec, + legacy: bool, +) -> Result<(EndpointServerPools, SetupType), Error> { + if pool_args.is_empty() { + return Err(Error::msg("无效参数")); + } + + let (pooleps, setup_type) = create_pool_endpoints(server_addr, pool_args)?; + + let mut ret = EndpointServerPools::new(); + + for (i, eps) in pooleps.iter().enumerate() { + let ep = PoolEndpoints { + legacy: legacy, + set_count: pool_args[i].layout.len(), + drives_per_set: pool_args[i].layout[0].len(), + endpoints: eps.clone(), + cmd_line: pool_args[i].cmdline.clone(), + platform: String::new(), + }; + + ret.add(ep)?; + } + + Ok((ret, setup_type)) +} + +#[cfg(test)] +mod test { + + use crate::disks_layout::DisksLayout; + + use super::*; + + #[test] + fn test_create_server_endpoints() { + let cases = vec![( + ":9000", + vec![ + // "/Users/weisd/fs".to_string(), + "http://localhost:900{1...2}/export{1...64}".to_string(), + ], + )]; + + for (addr, args) in cases { + let layouts = DisksLayout::new(args).unwrap(); + + println!("layouts:{:?},{}", &layouts.pools, &layouts.legacy); + + let (server_pool, setup_type) = + create_server_endpoints(addr.to_string(), &layouts.pools, layouts.legacy).unwrap(); + + println!("setup_type -- {:?}", setup_type); + println!("server_pool == {:?}", server_pool); + } + + // create_server_endpoints(server_addr, pool_args, legacy) + } +} diff --git a/store/src/erasure.rs b/ecstore/src/erasure.rs similarity index 100% rename from store/src/erasure.rs rename to ecstore/src/erasure.rs diff --git a/store/src/error.rs b/ecstore/src/error.rs similarity index 100% rename from store/src/error.rs rename to ecstore/src/error.rs diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs new file mode 100644 index 00000000..4bcbcb3c --- /dev/null +++ b/ecstore/src/lib.rs @@ -0,0 +1,9 @@ +mod disks_layout; +mod ellipses; +mod endpoint; +mod erasure; +mod error; +pub mod s3; +pub mod store; +mod stream; +mod utils; diff --git a/ecstore/src/s3.rs b/ecstore/src/s3.rs new file mode 100644 index 00000000..464ccb62 --- /dev/null +++ b/ecstore/src/s3.rs @@ -0,0 +1,276 @@ +use s3s::dto::*; +use s3s::s3_error; +use s3s::S3Result; +use s3s::S3; +use s3s::{S3Request, S3Response}; + +use crate::store::ECStore; + +#[async_trait::async_trait] +impl S3 for ECStore { + #[tracing::instrument] + async fn create_bucket( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + let output = CreateBucketOutput::default(); // TODO: handle other fields + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn copy_object( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + let (bucket, key) = match input.copy_source { + CopySource::AccessPoint { .. } => return Err(s3_error!(NotImplemented)), + CopySource::Bucket { + ref bucket, + ref key, + .. + } => (bucket, key), + }; + + let output = CopyObjectOutput { + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn delete_bucket( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + Ok(S3Response::new(DeleteBucketOutput {})) + } + + #[tracing::instrument] + async fn delete_object( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + let output = DeleteObjectOutput::default(); // TODO: handle other fields + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn delete_objects( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + let output = DeleteObjectsOutput { + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn get_bucket_location( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + let output = GetBucketLocationOutput::default(); + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn get_object( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + let output = GetObjectOutput { + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn head_bucket( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + Ok(S3Response::new(HeadBucketOutput::default())) + } + + #[tracing::instrument] + async fn head_object( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + let output = HeadObjectOutput { + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn list_buckets( + &self, + _: S3Request, + ) -> S3Result> { + let output = ListBucketsOutput { + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn list_objects( + &self, + req: S3Request, + ) -> S3Result> { + let v2_resp = self.list_objects_v2(req.map_input(Into::into)).await?; + + Ok(v2_resp.map_output(|v2| ListObjectsOutput { + contents: v2.contents, + delimiter: v2.delimiter, + encoding_type: v2.encoding_type, + name: v2.name, + prefix: v2.prefix, + max_keys: v2.max_keys, + ..Default::default() + })) + } + + #[tracing::instrument] + async fn list_objects_v2( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + let output = ListObjectsV2Output { + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn put_object( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + let output = PutObjectOutput { + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn create_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + let output = CreateMultipartUploadOutput { + ..Default::default() + }; + + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn upload_part( + &self, + req: S3Request, + ) -> S3Result> { + let UploadPartInput { + body, + upload_id, + part_number, + .. + } = req.input; + + let output = UploadPartOutput { + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn upload_part_copy( + &self, + req: S3Request, + ) -> S3Result> { + let input = req.input; + + let output = UploadPartCopyOutput { + ..Default::default() + }; + + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn list_parts( + &self, + req: S3Request, + ) -> S3Result> { + let ListPartsInput { + bucket, + key, + upload_id, + .. + } = req.input; + + let output = ListPartsOutput { + bucket: Some(bucket), + key: Some(key), + upload_id: Some(upload_id), + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn complete_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + let CompleteMultipartUploadInput { + multipart_upload, + bucket, + key, + upload_id, + .. + } = req.input; + + let output = CompleteMultipartUploadOutput { + bucket: Some(bucket), + key: Some(key), + ..Default::default() + }; + Ok(S3Response::new(output)) + } + + #[tracing::instrument] + async fn abort_multipart_upload( + &self, + req: S3Request, + ) -> S3Result> { + Ok(S3Response::new(AbortMultipartUploadOutput { + ..Default::default() + })) + } +} diff --git a/store/src/store.rs b/ecstore/src/store.rs similarity index 58% rename from store/src/store.rs rename to ecstore/src/store.rs index 708a3a0f..f73cd245 100644 --- a/store/src/store.rs +++ b/ecstore/src/store.rs @@ -1,18 +1,28 @@ use super::endpoint::Endpoint; +use crate::endpoint::EndpointServerPools; -pub struct Store { +use std::fmt::Debug; + +#[derive(Debug)] +pub struct ECStore { pub id: uuid::Uuid, pub disks: Vec>, pub pools: Vec, pub peer: Vec, } -impl Store {} +impl ECStore { + pub fn new(endpoints: EndpointServerPools) { + unimplemented!() + } +} +#[derive(Debug)] pub struct Sets { pub sets: Vec, } +#[derive(Debug)] pub struct Objects { pub endpoints: Vec, pub disks: Vec, @@ -22,6 +32,7 @@ pub struct Objects { pub default_parity_count: usize, } -trait DiskAPI {} +#[async_trait::async_trait] +trait DiskAPI: Debug + Send + Sync + 'static {} pub trait StorageAPI {} diff --git a/store/src/stream.rs b/ecstore/src/stream.rs similarity index 100% rename from store/src/stream.rs rename to ecstore/src/stream.rs diff --git a/ecstore/src/utils/mod.rs b/ecstore/src/utils/mod.rs new file mode 100644 index 00000000..5f521a38 --- /dev/null +++ b/ecstore/src/utils/mod.rs @@ -0,0 +1,2 @@ +pub mod net; +pub mod string; diff --git a/ecstore/src/utils/net.rs b/ecstore/src/utils/net.rs new file mode 100644 index 00000000..6730f4d8 --- /dev/null +++ b/ecstore/src/utils/net.rs @@ -0,0 +1,94 @@ +use std::{ + collections::HashMap, + net::{IpAddr, ToSocketAddrs}, +}; + +use anyhow::Error; +use netif; +use url::Host; + +pub fn split_host_port(s: &str) -> Result<(String, u16), Error> { + let parts: Vec<&str> = s.split(':').collect(); + if parts.len() == 2 { + if let Ok(port) = parts[1].parse::() { + return Ok((parts[0].to_string(), port)); + } + } + Err(Error::msg("Invalid address format or port number")) +} + +// is_local_host 判断是否是本地ip +pub fn is_local_host(host: Host<&str>, port: u16, local_port: u16) -> bool { + let local_ips = must_get_local_ips(); + + let local_map = + local_ips + .iter() + .map(|ip| ip.to_string()) + .fold(HashMap::new(), |mut acc, item| { + *acc.entry(item).or_insert(true) = true; + acc + }); + + let is_local_host = match host { + Host::Domain(domain) => { + let ips: Vec = (domain, 0) + .to_socket_addrs() + .unwrap_or(Vec::new().into_iter()) + .map(|addr| addr.ip().to_string()) + .collect(); + + let mut isok = false; + for ip in ips.iter() { + if local_map.contains_key(ip) { + isok = true; + break; + } + } + isok + } + Host::Ipv4(ip) => local_map.contains_key(&ip.to_string()), + Host::Ipv6(ip) => local_map.contains_key(&ip.to_string()), + }; + + if port > 0 { + return is_local_host && port == local_port; + } + + is_local_host +} + +pub fn must_get_local_ips() -> Vec { + let mut v: Vec = Vec::new(); + if let Some(up) = netif::up().ok() { + v = up.map(|x| x.address().to_owned()).collect(); + } + + v +} + +#[cfg(test)] +mod test { + use std::net::Ipv4Addr; + + use super::*; + + #[test] + fn test_must_get_local_ips() { + let ips = must_get_local_ips(); + for ip in ips.iter() { + println!("{:?}", ip) + } + } + + #[test] + fn test_is_local_host() { + // let host = Host::Ipv4(Ipv4Addr::new(192, 168, 0, 233)); + let host = Host::Ipv4(Ipv4Addr::new(127, 0, 0, 1)); + // let host = Host::Domain("localhost"); + let port = 0; + let local_port = 9000; + let is = is_local_host(host, port, local_port); + assert!(is) + } +} diff --git a/ecstore/src/utils/string.rs b/ecstore/src/utils/string.rs new file mode 100644 index 00000000..1f2dbc60 --- /dev/null +++ b/ecstore/src/utils/string.rs @@ -0,0 +1,133 @@ +use std::collections::HashMap; +use std::fmt; + +#[derive(Debug, Clone)] +pub struct StringSet(HashMap); + +impl StringSet { + // ToSlice - returns StringSet as a vector of strings. + pub fn to_slice(&self) -> Vec { + let mut keys = self.0.keys().cloned().collect::>(); + keys.sort(); + keys + } + + // IsEmpty - returns whether the set is empty or not. + pub fn is_empty(&self) -> bool { + self.0.len() == 0 + } + + // Add - adds a string to the set. + pub fn add(&mut self, s: String) { + self.0.insert(s, ()); + } + + // Remove - removes a string from the set. It does nothing if the string does not exist in the set. + pub fn remove(&mut self, s: &str) { + self.0.remove(s); + } + + // Contains - checks if a string is in the set. + pub fn contains(&self, s: &str) -> bool { + self.0.contains_key(s) + } + + // FuncMatch - returns a new set containing each value that passes the match function. + pub fn func_match(&self, match_fn: F, match_string: &str) -> StringSet + where + F: Fn(&str, &str) -> bool, + { + StringSet( + self.0 + .iter() + .filter(|(k, _)| match_fn(k, match_string)) + .map(|(k, _)| (k.clone(), ())) + .collect::>(), + ) + } + + // ApplyFunc - returns a new set containing each value processed by 'apply_fn'. + pub fn apply_func(&self, apply_fn: F) -> StringSet + where + F: Fn(&str) -> String, + { + StringSet( + self.0 + .iter() + .map(|(k, _)| (apply_fn(k), ())) + .collect::>(), + ) + } + + // Equals - checks whether the given set is equal to the current set or not. + pub fn equals(&self, other: &StringSet) -> bool { + if self.0.len() != other.0.len() { + return false; + } + self.0.iter().all(|(k, _)| other.0.contains_key(k)) + } + + // Intersection - returns the intersection with the given set as a new set. + pub fn intersection(&self, other: &StringSet) -> StringSet { + StringSet( + self.0 + .iter() + .filter(|(k, _)| other.0.contains_key::(k)) + .map(|(k, _)| (k.clone(), ())) + .collect::>(), + ) + } + + // Difference - returns the difference with the given set as a new set. + pub fn difference(&self, other: &StringSet) -> StringSet { + StringSet( + self.0 + .iter() + .filter(|(k, _)| !other.0.contains_key::(k)) + .map(|(k, _)| (k.clone(), ())) + .collect::>(), + ) + } + + // Union - returns the union with the given set as a new set. + pub fn union(&self, other: &StringSet) -> StringSet { + let mut new_set = self.clone(); + for (k, _) in other.0.iter() { + new_set.0.insert(k.clone(), ()); + } + new_set + } +} + +// Implementing JSON serialization and deserialization would require the serde crate. +// You would also need to implement Display and PartialEq traits for more idiomatic Rust. + +// Implementing Display trait to provide a string representation of the set. +impl fmt::Display for StringSet { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "{}", self.to_slice().join(", ")) + } +} + +// Implementing PartialEq and Eq traits to allow comparison of StringSet instances. +impl PartialEq for StringSet { + fn eq(&self, other: &StringSet) -> bool { + self.equals(other) + } +} + +impl Eq for StringSet {} + +// NewStringSet - creates a new string set. +pub fn new_string_set() -> StringSet { + StringSet(HashMap::new()) +} + +// CreateStringSet - creates a new string set with given string values. +pub fn create_string_set(sl: Vec) -> StringSet { + let mut set = new_string_set(); + for k in sl { + set.add(k); + } + set +} diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 122a1c78..662d180d 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -8,7 +8,24 @@ rust-version.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[[bin]] +name = "rustfs" +required-features = ["binary"] + + +[features] +binary = ["tokio/full", "dep:clap", "dep:tracing-subscriber", "dep:hyper-util"] + [dependencies] -store = { path = "../store" } -tracing-subscriber = { version = "0.3.18" } -time = { workspace = true, features = ["parsing", "formatting"] } +ecstore = { path = "../ecstore" } +# time = { workspace = true, features = ["parsing", "formatting"] } +# s3s = "0.10.0" +hyper-util = { version = "0.1.5", optional = true, features = ["server-auto", "server-graceful", "http1", "http2", "tokio"] } +tokio.workspace = true +clap = { version = "4.5.7", optional = true, features = ["derive"] } +# tracing.workspace = true +# tokio-util.workspace = true +tracing-subscriber = { version = "0.3.18", optional = true, features = ["env-filter", "time"] } +s3s = "0.10.0" +# async-trait = "0.1.80" diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index e7a11a96..108da2e2 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -1,3 +1,160 @@ -fn main() { - println!("Hello, world!"); +#![forbid(unsafe_code)] +#![deny(clippy::all, clippy::pedantic)] + +use s3s_fs::FileSystem; +use s3s_fs::Result; + +use s3s::auth::SimpleAuth; +use s3s::service::S3ServiceBuilder; + +use std::io::IsTerminal; +use std::path::PathBuf; + +use tokio::net::TcpListener; + +use clap::{CommandFactory, Parser}; +use tracing::info; + +use hyper_util::rt::{TokioExecutor, TokioIo}; +use hyper_util::server::conn::auto::Builder as ConnBuilder; + +#[derive(Debug, Parser)] +#[command(version)] +struct Opt { + /// Host name to listen on. + #[arg(long, default_value = "localhost")] + host: String, + + /// Port number to listen on. + #[arg(long, default_value = "8014")] // The original design was finished on 2020-08-14. + port: u16, + + /// Access key used for authentication. + #[arg(long)] + access_key: Option, + + /// Secret key used for authentication. + #[arg(long)] + secret_key: Option, + + /// Domain name used for virtual-hosted-style requests. + #[arg(long)] + domain_name: Option, + + /// Root directory of stored data. + root: PathBuf, +} + +fn setup_tracing() { + use tracing_subscriber::EnvFilter; + + let env_filter = EnvFilter::from_default_env(); + let enable_color = std::io::stdout().is_terminal(); + + tracing_subscriber::fmt() + .pretty() + .with_env_filter(env_filter) + .with_ansi(enable_color) + .init(); +} + +fn check_cli_args(opt: &Opt) { + use clap::error::ErrorKind; + + let mut cmd = Opt::command(); + + // TODO: how to specify the requirements with clap derive API? + if let (Some(_), None) | (None, Some(_)) = (&opt.access_key, &opt.secret_key) { + let msg = "access key and secret key must be specified together"; + cmd.error(ErrorKind::MissingRequiredArgument, msg).exit(); + } + + if let Some(ref s) = opt.domain_name { + if s.contains('/') { + let msg = format!("expected domain name, found URL-like string: {s:?}"); + cmd.error(ErrorKind::InvalidValue, msg).exit(); + } + } +} + +fn main() -> Result { + let opt = Opt::parse(); + check_cli_args(&opt); + + setup_tracing(); + + run(opt) +} + +#[tokio::main] +async fn run(opt: Opt) -> Result { + // Setup S3 provider + let fs = FileSystem::new(opt.root)?; + + // Setup S3 service + let service = { + let mut b = S3ServiceBuilder::new(fs); + + // Enable authentication + if let (Some(ak), Some(sk)) = (opt.access_key, opt.secret_key) { + b.set_auth(SimpleAuth::from_single(ak, sk)); + info!("authentication is enabled"); + } + + // Enable parsing virtual-hosted-style requests + if let Some(domain_name) = opt.domain_name { + b.set_base_domain(domain_name); + info!("virtual-hosted-style requests are enabled"); + } + + b.build() + }; + + // Run server + let listener = TcpListener::bind((opt.host.as_str(), opt.port)).await?; + let local_addr = listener.local_addr()?; + + let hyper_service = service.into_shared(); + + let http_server = ConnBuilder::new(TokioExecutor::new()); + let graceful = hyper_util::server::graceful::GracefulShutdown::new(); + + let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c()); + + info!("server is running at http://{local_addr}"); + + loop { + let (socket, _) = tokio::select! { + res = listener.accept() => { + match res { + Ok(conn) => conn, + Err(err) => { + tracing::error!("error accepting connection: {err}"); + continue; + } + } + } + _ = ctrl_c.as_mut() => { + break; + } + }; + + let conn = http_server.serve_connection(TokioIo::new(socket), hyper_service.clone()); + let conn = graceful.watch(conn.into_owned()); + tokio::spawn(async move { + let _ = conn.await; + }); + } + + tokio::select! { + () = graceful.shutdown() => { + tracing::debug!("Gracefully shutdown!"); + }, + () = tokio::time::sleep(std::time::Duration::from_secs(10)) => { + tracing::debug!("Waited 10 seconds for graceful shutdown, aborting..."); + } + } + + info!("server is stopped"); + Ok(()) } diff --git a/store/src/endpoint.rs b/store/src/endpoint.rs deleted file mode 100644 index bf2d1524..00000000 --- a/store/src/endpoint.rs +++ /dev/null @@ -1,22 +0,0 @@ -pub struct EndpointServerPools(Vec); - -pub struct PoolEndpoints { - // indicates if endpoints are provided in non-ellipses style - pub legacy: bool, - pub set_count: usize, - pub drives_per_set: usize, - pub endpoints: Endpoints, - pub cmd_line: String, - pub platform: String, -} - -pub struct Endpoints(Vec); - -pub struct Endpoint { - pub url: url::Url, - - pub is_local: bool, - pub pool_idx: i32, - pub set_idx: i32, - pub disk_idx: i32, -} diff --git a/store/src/lib.rs b/store/src/lib.rs deleted file mode 100644 index 58fc360b..00000000 --- a/store/src/lib.rs +++ /dev/null @@ -1,5 +0,0 @@ -mod endpoint; -mod erasure; -mod error; -mod store; -mod stream;