From a8d9970500da772bbce1ed515640b590434aaffc Mon Sep 17 00:00:00 2001 From: loverustfs <155562731+loverustfs@users.noreply.github.com> Date: Wed, 21 Aug 2024 14:59:29 +0800 Subject: [PATCH 01/23] Update TODO.md --- TODO.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/TODO.md b/TODO.md index b15b6097..973bef0a 100644 --- a/TODO.md +++ b/TODO.md @@ -40,3 +40,5 @@ - [ ] 版本控制 - [ ] 对象锁 - [ ] 修复 +- [ ] 桶配额 +- [ ] 桶只读 From d183dd6e9184fa8aaffcd3e26c62a58f8938d56b Mon Sep 17 00:00:00 2001 From: loverustfs <155562731+loverustfs@users.noreply.github.com> Date: Wed, 21 Aug 2024 23:42:42 +0800 Subject: [PATCH 02/23] Update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 2f791d6e..0dc3785f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ /target .DS_Store .idea +.vscode From 0a373bc260afd1c9a8a3153b60ff888088927439 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 27 Aug 2024 11:40:00 +0800 Subject: [PATCH 03/23] =?UTF-8?q?=E9=80=9A=E8=BF=87tonic=E6=94=AF=E6=8C=81?= =?UTF-8?q?=E8=8A=82=E7=82=B9=E9=97=B4=E9=80=9A=E4=BF=A1=EF=BC=88=E5=88=9D?= =?UTF-8?q?=E6=AD=A5?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Cargo.lock | 629 +++++++++++++++++- Cargo.toml | 45 +- common/protos/Cargo.toml | 17 + common/protos/build.rs | 258 +++++++ .../protos/src/flatbuffers_generated/mod.rs | 1 + .../src/flatbuffers_generated/models.rs | 124 ++++ common/protos/src/lib.rs | 6 + common/protos/src/models.fbs | 5 + common/protos/src/node.proto | 19 + common/protos/src/proto_gen/mod.rs | 1 + common/protos/src/proto_gen/node_service.rs | 250 +++++++ e2e_test/Cargo.toml | 15 + e2e_test/README.md | 0 e2e_test/src/lib.rs | 1 + e2e_test/src/reliant/README.md | 1 + e2e_test/src/reliant/mod.rs | 1 + e2e_test/src/reliant/node_interact_test.rs | 46 ++ rustfs/Cargo.toml | 43 +- rustfs/src/grpc.rs | 47 ++ rustfs/src/main.rs | 9 +- rustfs/src/service.rs | 150 +++++ 21 files changed, 1632 insertions(+), 36 deletions(-) create mode 100644 common/protos/Cargo.toml create mode 100644 common/protos/build.rs create mode 100644 common/protos/src/flatbuffers_generated/mod.rs create mode 100644 common/protos/src/flatbuffers_generated/models.rs create mode 100644 common/protos/src/lib.rs create mode 100644 common/protos/src/models.fbs create mode 100644 common/protos/src/node.proto create mode 100644 common/protos/src/proto_gen/mod.rs create mode 100644 common/protos/src/proto_gen/node_service.rs create mode 100644 e2e_test/Cargo.toml create mode 100644 e2e_test/README.md create mode 100644 e2e_test/src/lib.rs create mode 100644 e2e_test/src/reliant/README.md create mode 100644 e2e_test/src/reliant/mod.rs create mode 100644 e2e_test/src/reliant/node_interact_test.rs create mode 100644 rustfs/src/grpc.rs create mode 100644 rustfs/src/service.rs diff --git a/Cargo.lock b/Cargo.lock index dd42ff16..ce909a40 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17,6 +17,12 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" +[[package]] +name = "adler2" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" + [[package]] name = "ahash" version = "0.7.8" @@ -86,12 +92,40 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "anyhow" +version = "1.0.86" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" + [[package]] name = "arrayvec" version = "0.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96d30a06541fbafbc7f82ed10c06164cfbd2c401138f6addd8404629c4b16711" +[[package]] +name = "async-stream" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd56dd203fef61ac097dd65721a419ddccb106b2d2b70ba60a6b529f03961a51" +dependencies = [ + "async-stream-impl", + "futures-core", + "pin-project-lite", +] + +[[package]] +name = "async-stream-impl" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16e62a023e7c117e27523144c5d2459f4397fcc3cab0085af8e2224f643a0193" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "async-trait" version = "0.1.80" @@ -124,6 +158,53 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c4b4d0bd25bd0b74681c0ad21497610ce1b7c91b1022cd21c80c6fbdd9476b0" +[[package]] +name = "axum" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a6c9af12842a67734c9a2e355436e5d03b22383ed60cf13cd0c18fbfe3dcbcf" +dependencies = [ + "async-trait", + "axum-core", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "itoa", + "matchit", + "memchr", + "mime", + "percent-encoding", + "pin-project-lite", + "rustversion", + "serde", + "sync_wrapper 1.0.1", + "tower", + "tower-layer", + "tower-service", +] + +[[package]] +name = "axum-core" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a15c63fd72d41492dc4f497196f5da1fb04fb7529e631d73630d1b491e47a2e3" +dependencies = [ + "async-trait", + "bytes", + "futures-util", + "http", + "http-body", + "http-body-util", + "mime", + "pin-project-lite", + "rustversion", + "sync_wrapper 0.1.2", + "tower-layer", + "tower-service", +] + [[package]] name = "backtrace" version = "0.3.73" @@ -134,11 +215,17 @@ dependencies = [ "cc", "cfg-if", "libc", - "miniz_oxide", + "miniz_oxide 0.7.4", "object", "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.22.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" + [[package]] name = "base64-simd" version = "0.8.0" @@ -267,6 +354,15 @@ dependencies = [ "libc", ] +[[package]] +name = "crc32c" +version = "0.6.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" +dependencies = [ + "rustc_version", +] + [[package]] name = "crc32fast" version = "1.4.2" @@ -307,6 +403,16 @@ dependencies = [ "subtle", ] +[[package]] +name = "e2e_test" +version = "0.0.1" +dependencies = [ + "flatbuffers", + "protos", + "tokio", + "tonic", +] + [[package]] name = "ecstore" version = "0.1.0" @@ -346,12 +452,60 @@ dependencies = [ "xxhash-rust", ] +[[package]] +name = "either" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60b1af1c220855b6ceac025d3f6ecdd2b7c4894bfe9cd9bda4fbb4bc7c0d4cf0" + [[package]] name = "equivalent" version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" +[[package]] +name = "errno" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "534c5cf6194dfab3db3242765c03bbe257cf92f22b38f6bc0c58d59108a820ba" +dependencies = [ + "libc", + "windows-sys 0.52.0", +] + +[[package]] +name = "fastrand" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fc0510504f03c51ada170672ac806f1f105a88aa97a5281117e1ddc3368e51a" + +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + +[[package]] +name = "flatbuffers" +version = "24.3.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8add37afff2d4ffa83bc748a70b4b1370984f6980768554182424ef71447c35f" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + +[[package]] +name = "flate2" +version = "1.0.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c0596c1eac1f9e04ed902702e9878208b336edc9d6fddc8a48387349bab3666" +dependencies = [ + "crc32fast", + "miniz_oxide 0.8.0", +] + [[package]] name = "fnv" version = "1.0.7" @@ -510,7 +664,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap", + "indexmap 2.2.6", "slab", "tokio", "tokio-util", @@ -627,6 +781,20 @@ dependencies = [ "pin-project-lite", "smallvec", "tokio", + "want", +] + +[[package]] +name = "hyper-timeout" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3203a961e5c83b6f5498933e78b6b263e208c197b63e9c6c53cc82ffd3f63793" +dependencies = [ + "hyper", + "hyper-util", + "pin-project-lite", + "tokio", + "tower-service", ] [[package]] @@ -636,12 +804,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7b875924a60b96e5d7b9ae7b066540b1dd1cbd90d1828f54c92e02a283351c56" dependencies = [ "bytes", + "futures-channel", "futures-util", "http", "http-body", "hyper", "pin-project-lite", + "socket2", "tokio", + "tower", + "tower-service", + "tracing", ] [[package]] @@ -654,6 +827,16 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", +] + [[package]] name = "indexmap" version = "2.2.6" @@ -679,6 +862,15 @@ version = "1.70.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8478577c03552c21db0e2724ffb8986a5ce7af88107e6be5d2ee6e158c12800" +[[package]] +name = "itertools" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.11" @@ -703,6 +895,12 @@ version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4ec2a862134d2a7d32d7983ddcdd1c4923530833c9f2ea1a44fc5fa473989058" +[[package]] +name = "linux-raw-sys" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" + [[package]] name = "lock_api" version = "0.4.12" @@ -737,6 +935,12 @@ dependencies = [ "regex-automata 0.1.10", ] +[[package]] +name = "matchit" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94" + [[package]] name = "memchr" version = "2.7.4" @@ -764,6 +968,15 @@ dependencies = [ "adler", ] +[[package]] +name = "miniz_oxide" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2d80299ef12ff69b16a84bb182e3b9df68b5a91574d3d4fa6e41b65deec4df1" +dependencies = [ + "adler2", +] + [[package]] name = "mio" version = "0.8.11" @@ -775,6 +988,12 @@ dependencies = [ "windows-sys 0.48.0", ] +[[package]] +name = "multimap" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5ce46fe64a9d73be07dcbe690a38ce1b293be448fd8ce1e6c1b8062c9f72c6a" + [[package]] name = "netif" version = "0.1.6" @@ -839,6 +1058,12 @@ dependencies = [ "libc", ] +[[package]] +name = "numeric_cast" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf70ee2d9b1737d1836c20d9f8f96ec3901b2bf92128439db13237ddce9173a5" + [[package]] name = "object" version = "0.36.0" @@ -965,6 +1190,36 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.2.6", +] + +[[package]] +name = "pin-project" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bf43b791c5b9e34c3d182969b4abb522f9343702850a2e57f460d00d09b4b3" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2f38a4412a78282e09a2cf38d195ea5420d15ba0602cb375210efbc877243965" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "pin-project-lite" version = "0.2.14" @@ -995,6 +1250,16 @@ version = "0.2.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" +[[package]] +name = "prettyplease" +version = "0.2.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479cf940fbbb3426c32c5d5176f62ad57549a0bb84773423ba8be9d089f5faba" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.86" @@ -1005,10 +1270,97 @@ dependencies = [ ] [[package]] -name = "quick-xml" -version = "0.31.0" +name = "prost" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1004a344b30a54e2ee58d66a71b32d2db2feb0a31f9a2d302bf0536f15de2a33" +checksum = "e13db3d3fde688c61e2446b4d843bc27a7e8af269a69440c0308021dc92333cc" +dependencies = [ + "bytes", + "prost-derive", +] + +[[package]] +name = "prost-build" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5bb182580f71dd070f88d01ce3de9f4da5021db7115d2e1c3605a754153b77c1" +dependencies = [ + "bytes", + "heck", + "itertools", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost", + "prost-types", + "regex", + "syn", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18bec9b0adc4eba778b33684b7ba3e7137789434769ee3ce3930463ef904cfca" +dependencies = [ + "anyhow", + "itertools", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "prost-types" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cee5168b05f49d4b0ca581206eb14a7b22fafd963efe729ac48eb03266e25cc2" +dependencies = [ + "prost", +] + +[[package]] +name = "protobuf" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bcc343da15609eaecd65f8aa76df8dc4209d325131d8219358c0aaaebab0bf6" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror", +] + +[[package]] +name = "protobuf-support" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0766e3675a627c327e4b3964582594b0e8741305d628a98a5de75a1d15f99b9" +dependencies = [ + "thiserror", +] + +[[package]] +name = "protos" +version = "0.0.1" +dependencies = [ + "flatbuffers", + "prost", + "prost-build", + "protobuf", + "tokio", + "tonic", + "tonic-build", + "tower", +] + +[[package]] +name = "quick-xml" +version = "0.36.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "96a05e2e8efddfa51a84ca47cec303fac86c8541b686d37cac5efc0e094417bc" dependencies = [ "memchr", "serde", @@ -1119,6 +1471,21 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" +[[package]] +name = "ring" +version = "0.17.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d" +dependencies = [ + "cc", + "cfg-if", + "getrandom", + "libc", + "spin", + "untrusted", + "windows-sys 0.52.0", +] + [[package]] name = "rmp" version = "0.8.14" @@ -1147,6 +1514,15 @@ version = "0.1.24" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "719b953e2095829ee67db738b3bfa9fa368c94900df327b3f07fe6e794d2fe1f" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "rustfs" version = "0.1.0" @@ -1155,21 +1531,95 @@ dependencies = [ "bytes", "clap", "ecstore", + "flatbuffers", "futures", "futures-util", "http", + "http-body", + "hyper", "hyper-util", "mime", "netif", + "pin-project-lite", + "prost", + "prost-build", + "prost-types", + "protobuf", + "protos", "s3s", "time", "tokio", + "tonic", + "tonic-build", + "tonic-reflection", + "tower", "tracing", "tracing-error", "tracing-subscriber", "transform-stream", ] +[[package]] +name = "rustix" +version = "0.38.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +dependencies = [ + "bitflags 2.6.0", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.52.0", +] + +[[package]] +name = "rustls" +version = "0.23.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pemfile" +version = "2.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "196fe16b00e106300d3e45ecfcb764fa292a535d7326a29a5875c579c7417425" +dependencies = [ + "base64", + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0a2ce646f8655401bb81e7927b812614bd5d91dbc968696be50603510fcaf0" + +[[package]] +name = "rustls-webpki" +version = "0.102.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + +[[package]] +name = "rustversion" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955d28af4278de8121b7ebeb796b6a45735dc01436d898801014aced2773a3d6" + [[package]] name = "ryu" version = "1.0.18" @@ -1178,9 +1628,9 @@ checksum = "f3cb5ba0dc43242ce17de99c180e96db90b235b8a9fdc9543c96d2209116bd9f" [[package]] name = "s3s" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67e6cdc8002708b435946eec39afa13c43e4288d1de6316a12816e4cfaaa6c2c" +checksum = "fa54e3b4b4791c8c62291516997866b4f265c3fcbfdbcdd0b8da62896fba8bfa" dependencies = [ "arrayvec", "async-trait", @@ -1189,7 +1639,9 @@ dependencies = [ "bytes", "bytestring", "chrono", + "crc32c", "crc32fast", + "digest", "futures", "hex-simd", "hmac", @@ -1202,6 +1654,7 @@ dependencies = [ "mime", "nom", "nugine-rust-utils", + "numeric_cast", "pin-project-lite", "quick-xml", "serde", @@ -1209,8 +1662,11 @@ dependencies = [ "sha1", "sha2", "smallvec", + "sync_wrapper 1.0.1", "thiserror", "time", + "tokio", + "tower", "tracing", "transform-stream", "urlencoding", @@ -1223,6 +1679,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61697e0a1c7e512e84a621326239844a24d8207b4669b41bc18b32ea5cbf988b" + [[package]] name = "serde" version = "1.0.203" @@ -1363,15 +1825,40 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "2.0.68" +version = "2.0.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "901fa70d88b9d6c98022e23b4136f9f3e54e4662c3bc1bd1d84a42a9a0f0c1e9" +checksum = "578e081a14e0cefc3279b0472138c513f37b41a08d5a3cca9b6e4e8ceb6cd525" dependencies = [ "proc-macro2", "quote", "unicode-ident", ] +[[package]] +name = "sync_wrapper" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2047c6ded9c721764247e62cd3b03c09ffc529b2ba5b10ec482ae507a4a70160" + +[[package]] +name = "sync_wrapper" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a7065abeca94b6a8a577f9bd45aa0867a2238b74e8eb67cf10d492bc39351394" + +[[package]] +name = "tempfile" +version = "3.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fcd239983515c23a32fb82099f97d0b11b8c72f654ed659363a95c3dad7a53" +dependencies = [ + "cfg-if", + "fastrand", + "once_cell", + "rustix", + "windows-sys 0.52.0", +] + [[package]] name = "thiserror" version = "1.0.61" @@ -1477,6 +1964,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-rustls" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" +dependencies = [ + "rustls", + "rustls-pki-types", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.15" @@ -1501,12 +1999,104 @@ dependencies = [ "tokio", ] +[[package]] +name = "tonic" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38659f4a91aba8598d27821589f5db7dddd94601e7a01b1e485a50e5484c7401" +dependencies = [ + "async-stream", + "async-trait", + "axum", + "base64", + "bytes", + "flate2", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-timeout", + "hyper-util", + "percent-encoding", + "pin-project", + "prost", + "rustls-pemfile", + "socket2", + "tokio", + "tokio-rustls", + "tokio-stream", + "tower", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tonic-build" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "568392c5a2bd0020723e3f387891176aabafe36fd9fcd074ad309dfa0c8eb964" +dependencies = [ + "prettyplease", + "proc-macro2", + "prost-build", + "quote", + "syn", +] + +[[package]] +name = "tonic-reflection" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b742c83ad673e9ab5b4ce0981f7b9e8932be9d60e8682cbf9120494764dbc173" +dependencies = [ + "prost", + "prost-types", + "tokio", + "tokio-stream", + "tonic", +] + +[[package]] +name = "tower" +version = "0.4.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" +dependencies = [ + "futures-core", + "futures-util", + "indexmap 1.9.3", + "pin-project", + "pin-project-lite", + "rand", + "slab", + "tokio", + "tokio-util", + "tower-layer", + "tower-service", + "tracing", +] + +[[package]] +name = "tower-layer" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e" + +[[package]] +name = "tower-service" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3" + [[package]] name = "tracing" version = "0.1.40" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3523ab5a71916ccf420eebdf5521fcef02141234bbc0b8a49f2fdc4544364ef" dependencies = [ + "log", "pin-project-lite", "tracing-attributes", "tracing-core", @@ -1582,6 +2172,12 @@ dependencies = [ "futures-core", ] +[[package]] +name = "try-lock" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" + [[package]] name = "typenum" version = "1.17.0" @@ -1609,6 +2205,12 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "url" version = "2.5.2" @@ -1667,6 +2269,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64" +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" diff --git a/Cargo.toml b/Cargo.toml index 9420e9b3..0ec2e78d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,21 +1,42 @@ [workspace] resolver = "2" -members = ["rustfs", "ecstore"] +members = ["rustfs", "ecstore", "e2e_test", "common/protos"] [workspace.package] edition = "2021" license = "MIT OR Apache-2.0" repository = "https://github.com/rustfs/rustfs" rust-version = "1.75" +version = "0.0.1" [workspace.dependencies] +async-trait = "0.1.80" +bytes = "1.6.0" +clap = { version = "4.5.7", features = ["derive"] } +ecstore = { path = "./ecstore" } +flatbuffers = "24.3.25" +futures = "0.3.30" +futures-util = "0.3.30" +hyper = "1.3.1" +hyper-util = { version = "0.1.5", features = [ + "tokio", + "server-auto", + "server-graceful", +] } +http = "1.1.0" +http-body = "1.0.0" +mime = "0.3.17" +netif = "0.1.6" +pin-project-lite = "0.2" +# pin-utils = "0.1.0" +prost = "0.13.1" +prost-build = "0.13.1" +prost-types = "0.13.1" +protobuf = "3.2" +protos = { path = "./common/protos" } +s3s = { version = "0.10.1", default-features = true, features = ["tower"] } serde = { version = "1.0.203", features = ["derive"] } serde_json = "1.0.117" -tracing = "0.1.40" -tracing-error = "0.2.0" -futures = "0.3.30" -bytes = "1.6.0" -http = "1.1.0" thiserror = "1.0.61" time = { version = "0.3.36", features = [ "std", @@ -24,6 +45,12 @@ time = { version = "0.3.36", features = [ "macros", "serde", ] } -async-trait = "0.1.80" -tokio = { version = "1.38.0", features = ["fs"] } -futures-util = "0.3.30" +tokio = { version = "1.38.0", features = ["fs", "rt-multi-thread"] } +tonic = { version = "0.12.1", features = ["gzip"] } +tonic-build = "0.12.1" +tonic-reflection = "0.12" +tower = "0.4.13" +tracing = "0.1.40" +tracing-error = "0.2.0" +tracing-subscriber = { version = "0.3.18", features = ["env-filter", "time"] } +transform-stream = "0.3.0" \ No newline at end of file diff --git a/common/protos/Cargo.toml b/common/protos/Cargo.toml new file mode 100644 index 00000000..a6ab9df6 --- /dev/null +++ b/common/protos/Cargo.toml @@ -0,0 +1,17 @@ +[package] +name = "protos" +version.workspace = true +edition.workspace = true + +[dependencies] +#async-backtrace = { workspace = true, optional = true } +flatbuffers = { workspace = true } +prost = { workspace = true } +protobuf = { workspace = true } +tokio = { workspace = true } +tonic = { workspace = true, features = ["transport", "tls"] } +tower = { workspace = true } + +[build-dependencies] +prost-build = { workspace = true } +tonic-build = { workspace = true } diff --git a/common/protos/build.rs b/common/protos/build.rs new file mode 100644 index 00000000..35efacb7 --- /dev/null +++ b/common/protos/build.rs @@ -0,0 +1,258 @@ +use std::{ + cmp, env, fs, + io::Write, + path::{Path, PathBuf}, + process::Command, +}; + +type AnyError = Box; + +const ENV_OUT_DIR: &str = "OUT_DIR"; +const VERSION_PROTOBUF: Version = Version(27, 0, 0); // 27.0 +const VERSION_FLATBUFFERS: Version = Version(24, 3, 25); // 24.3.25 +/// Build protos if the major version of `flatc` or `protoc` is greater +/// or lesser than the expected version. +const ENV_BUILD_PROTOS: &str = "BUILD_PROTOS"; +/// Path of `flatc` binary. +const ENV_FLATC_PATH: &str = "FLATC_PATH"; + +fn main() -> Result<(), AnyError> { + let version = protobuf_compiler_version()?; + let need_compile = match version.compare_ext(&VERSION_PROTOBUF) { + Ok(cmp::Ordering::Equal) => true, + Ok(_) => { + let version_err = Version::build_error_message(&version, &VERSION_PROTOBUF).unwrap(); + println!("cargo:warning=Tool `protoc` {version_err}, skip compiling."); + false + } + Err(version_err) => { + // return Err(format!("Tool `protoc` {version_err}, please update it.").into()); + println!("cargo:warning=Tool `protoc` {version_err}, please update it."); + false + } + }; + + if !need_compile { + return Ok(()); + } + + // path of proto file + let project_root_dir = env::current_dir()?; + let proto_dir = project_root_dir.join("src"); + let proto_files = &["node.proto"]; + let proto_out_dir = project_root_dir.join("src").join("proto_gen"); + let flatbuffer_out_dir = project_root_dir.join("src").join("flatbuffers_generated"); + let descriptor_set_path = PathBuf::from(env::var(ENV_OUT_DIR).unwrap()).join("proto-descriptor.bin"); + + tonic_build::configure() + .out_dir(proto_out_dir) + .file_descriptor_set_path(descriptor_set_path) + .protoc_arg("--experimental_allow_proto3_optional") + .compile_well_known_types(true) + .emit_rerun_if_changed(false) + .compile(proto_files, &[proto_dir.clone()]) + .map_err(|e| format!("Failed to generate protobuf file: {e}."))?; + + // protos/gen/mod.rs + let generated_mod_rs_path = project_root_dir.join("src").join("proto_gen").join("mod.rs"); + + let mut generated_mod_rs = fs::File::create(generated_mod_rs_path)?; + writeln!(&mut generated_mod_rs, "pub mod node_service;")?; + generated_mod_rs.flush()?; + + let generated_mod_rs_path = project_root_dir.join("src").join("lib.rs"); + + let mut generated_mod_rs = fs::File::create(generated_mod_rs_path)?; + writeln!(&mut generated_mod_rs, "#![allow(unused_imports)]")?; + writeln!(&mut generated_mod_rs, "#![allow(clippy::all)]")?; + writeln!(&mut generated_mod_rs, "pub mod proto_gen;")?; + generated_mod_rs.flush()?; + + let flatc_path = match env::var(ENV_FLATC_PATH) { + Ok(path) => { + println!("cargo:warning=Specified flatc path by environment {ENV_FLATC_PATH}={path}"); + path + } + Err(_) => "flatc".to_string(), + }; + + // build src/protos/*.fbs files to src/protos/gen/ + compile_flatbuffers_models( + &mut generated_mod_rs, + &flatc_path, + proto_dir.clone(), + flatbuffer_out_dir.clone(), + vec!["models"], + )?; + Ok(()) +} + +/// Compile proto/**.fbs files. +fn compile_flatbuffers_models, S: AsRef>( + generated_mod_rs: &mut fs::File, + flatc_path: &str, + in_fbs_dir: P, + out_rust_dir: P, + mod_names: Vec, +) -> Result<(), AnyError> { + let version = flatbuffers_compiler_version(flatc_path)?; + let need_compile = match version.compare_ext(&VERSION_FLATBUFFERS) { + Ok(cmp::Ordering::Equal) => true, + Ok(_) => { + let version_err = Version::build_error_message(&version, &VERSION_FLATBUFFERS).unwrap(); + println!("cargo:warning=Tool `{flatc_path}` {version_err}, skip compiling."); + false + } + Err(version_err) => { + return Err(format!("Tool `{flatc_path}` {version_err}, please update it.").into()); + } + }; + + let fbs_dir = in_fbs_dir.as_ref(); + let rust_dir = out_rust_dir.as_ref(); + fs::create_dir_all(rust_dir)?; + + // $rust_dir/mod.rs + let mut sub_mod_rs = fs::File::create(rust_dir.join("mod.rs"))?; + writeln!(generated_mod_rs)?; + writeln!(generated_mod_rs, "mod flatbuffers_generated;")?; + for mod_name in mod_names.iter() { + let mod_name = mod_name.as_ref(); + writeln!(generated_mod_rs, "pub use flatbuffers_generated::{mod_name}::*;")?; + writeln!(&mut sub_mod_rs, "pub mod {mod_name};")?; + + if need_compile { + let fbs_file_path = fbs_dir.join(format!("{mod_name}.fbs")); + let output = Command::new(flatc_path) + .arg("-o") + .arg(rust_dir) + .arg("--rust") + .arg("--gen-mutable") + .arg("--gen-onefile") + .arg("--gen-name-strings") + .arg("--filename-suffix") + .arg("") + .arg(&fbs_file_path) + .output() + .map_err(|e| format!("Failed to execute process of flatc: {e}"))?; + if !output.status.success() { + return Err(format!( + "Failed to generate file '{}' by flatc(path: '{flatc_path}'): {}.", + fbs_file_path.display(), + String::from_utf8_lossy(&output.stderr), + ) + .into()); + } + } + } + generated_mod_rs.flush()?; + sub_mod_rs.flush()?; + + Ok(()) +} + +/// Run command `flatc --version` to get the version of flatc. +/// +/// ```ignore +/// $ flatc --version +/// flatc version 24.3.25 +/// ``` +fn flatbuffers_compiler_version(flatc_path: impl AsRef) -> Result { + let flatc_path = flatc_path.as_ref(); + Version::try_get(format!("{}", flatc_path.display()), |output| { + const PREFIX_OF_VERSION: &str = "flatc version "; + let output = output.trim(); + if let Some(version) = output.strip_prefix(PREFIX_OF_VERSION) { + Ok(version.to_string()) + } else { + Err(format!("Failed to get flatc version: {output}")) + } + }) +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Version(u32, u32, u32); + +impl Version { + fn try_get Result>(exe: String, output_to_version_string: F) -> Result { + let cmd = format!("{exe} --version"); + let output = std::process::Command::new(exe) + .arg("--version") + .output() + .map_err(|e| format!("Failed to execute `{cmd}`: {e}",))?; + let output_utf8 = String::from_utf8(output.stdout).map_err(|e| { + let output_lossy = String::from_utf8_lossy(e.as_bytes()); + format!("Command `{cmd}` returned invalid UTF-8('{output_lossy}'): {e}") + })?; + if output.status.success() { + let version_string = output_to_version_string(&output_utf8)?; + Ok(version_string.parse::()?) + } else { + Err(format!("Failed to get version by command `{cmd}`: {output_utf8}")) + } + } + + fn build_error_message(version: &Self, expected: &Self) -> Option { + match version.compare_major_version(expected) { + cmp::Ordering::Equal => None, + cmp::Ordering::Greater => Some(format!("version({version}) is greater than version({expected})")), + cmp::Ordering::Less => Some(format!("version({version}) is lesser than version({expected})")), + } + } + + fn compare_ext(&self, expected_version: &Self) -> Result { + match env::var(ENV_BUILD_PROTOS) { + Ok(build_protos) => { + if build_protos.is_empty() || build_protos == "0" { + Ok(self.compare_major_version(expected_version)) + } else { + match self.compare_major_version(expected_version) { + cmp::Ordering::Equal => Ok(cmp::Ordering::Equal), + _ => Err(Self::build_error_message(self, expected_version).unwrap()), + } + } + } + Err(_) => Ok(self.compare_major_version(expected_version)), + } + } + + fn compare_major_version(&self, other: &Self) -> cmp::Ordering { + self.0.cmp(&other.0) + } +} + +impl std::str::FromStr for Version { + type Err = String; + + fn from_str(s: &str) -> Result { + let mut version = [0_u32; 3]; + for (i, v) in s.split('.').take(3).enumerate() { + version[i] = v.parse().map_err(|e| format!("Failed to parse version string '{s}': {e}"))?; + } + Ok(Version(version[0], version[1], version[2])) + } +} + +impl std::fmt::Display for Version { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}.{}.{}", self.0, self.1, self.2) + } +} + +/// Run command `protoc --version` to get the version of flatc. +/// +/// ```ignore +/// $ protoc --version +/// libprotoc 27.0 +/// ``` +fn protobuf_compiler_version() -> Result { + Version::try_get("protoc".to_string(), |output| { + const PREFIX_OF_VERSION: &str = "libprotoc "; + let output = output.trim(); + if let Some(version) = output.strip_prefix(PREFIX_OF_VERSION) { + Ok(version.to_string()) + } else { + Err(format!("Failed to get protoc version: {output}")) + } + }) +} diff --git a/common/protos/src/flatbuffers_generated/mod.rs b/common/protos/src/flatbuffers_generated/mod.rs new file mode 100644 index 00000000..c446ac88 --- /dev/null +++ b/common/protos/src/flatbuffers_generated/mod.rs @@ -0,0 +1 @@ +pub mod models; diff --git a/common/protos/src/flatbuffers_generated/models.rs b/common/protos/src/flatbuffers_generated/models.rs new file mode 100644 index 00000000..e4949fdc --- /dev/null +++ b/common/protos/src/flatbuffers_generated/models.rs @@ -0,0 +1,124 @@ +// automatically generated by the FlatBuffers compiler, do not modify + +// @generated + +use core::cmp::Ordering; +use core::mem; + +extern crate flatbuffers; +use self::flatbuffers::{EndianScalar, Follow}; + +#[allow(unused_imports, dead_code)] +pub mod models { + + use core::cmp::Ordering; + use core::mem; + + extern crate flatbuffers; + use self::flatbuffers::{EndianScalar, Follow}; + + pub enum PingBodyOffset {} + #[derive(Copy, Clone, PartialEq)] + + pub struct PingBody<'a> { + pub _tab: flatbuffers::Table<'a>, + } + + impl<'a> flatbuffers::Follow<'a> for PingBody<'a> { + type Inner = PingBody<'a>; + #[inline] + unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { + _tab: flatbuffers::Table::new(buf, loc), + } + } + } + + impl<'a> PingBody<'a> { + pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; + + pub const fn get_fully_qualified_name() -> &'static str { + "models.PingBody" + } + + #[inline] + pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + PingBody { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + args: &'args PingBodyArgs<'args>, + ) -> flatbuffers::WIPOffset> { + let mut builder = PingBodyBuilder::new(_fbb); + if let Some(x) = args.payload { + builder.add_payload(x); + } + builder.finish() + } + + #[inline] + pub fn payload(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { + self._tab + .get::>>(PingBody::VT_PAYLOAD, None) + } + } + } + + impl flatbuffers::Verifiable for PingBody<'_> { + #[inline] + fn run_verifier(v: &mut flatbuffers::Verifier, pos: usize) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>>("payload", Self::VT_PAYLOAD, false)? + .finish(); + Ok(()) + } + } + pub struct PingBodyArgs<'a> { + pub payload: Option>>, + } + impl<'a> Default for PingBodyArgs<'a> { + #[inline] + fn default() -> Self { + PingBodyArgs { payload: None } + } + } + + pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + start_: flatbuffers::WIPOffset, + } + impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> { + #[inline] + pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset>) { + self.fbb_ + .push_slot_always::>(PingBody::VT_PAYLOAD, payload); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> { + let start = _fbb.start_table(); + PingBodyBuilder { + fbb_: _fbb, + start_: start, + } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } + } + + impl core::fmt::Debug for PingBody<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut ds = f.debug_struct("PingBody"); + ds.field("payload", &self.payload()); + ds.finish() + } + } +} // pub mod models diff --git a/common/protos/src/lib.rs b/common/protos/src/lib.rs new file mode 100644 index 00000000..4ab5a438 --- /dev/null +++ b/common/protos/src/lib.rs @@ -0,0 +1,6 @@ +#![allow(unused_imports)] +#![allow(clippy::all)] +pub mod proto_gen; + +mod flatbuffers_generated; +pub use flatbuffers_generated::models::*; diff --git a/common/protos/src/models.fbs b/common/protos/src/models.fbs new file mode 100644 index 00000000..d6a771ec --- /dev/null +++ b/common/protos/src/models.fbs @@ -0,0 +1,5 @@ +namespace models; + +table PingBody { + payload: [ubyte]; +} \ No newline at end of file diff --git a/common/protos/src/node.proto b/common/protos/src/node.proto new file mode 100644 index 00000000..879eb97f --- /dev/null +++ b/common/protos/src/node.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; +package node_service; + +/* -------------------------------------------------------------------- */ +message PingRequest { + uint64 version = 1; + bytes body = 2; +} + +message PingResponse { + uint64 version = 1; + bytes body = 2; +} + +/* -------------------------------------------------------------------- */ + +service NodeService { + rpc Ping(PingRequest) returns (PingResponse) {}; +} \ No newline at end of file diff --git a/common/protos/src/proto_gen/mod.rs b/common/protos/src/proto_gen/mod.rs new file mode 100644 index 00000000..35d3fe1b --- /dev/null +++ b/common/protos/src/proto_gen/mod.rs @@ -0,0 +1 @@ +pub mod node_service; diff --git a/common/protos/src/proto_gen/node_service.rs b/common/protos/src/proto_gen/node_service.rs new file mode 100644 index 00000000..463aac82 --- /dev/null +++ b/common/protos/src/proto_gen/node_service.rs @@ -0,0 +1,250 @@ +// This file is @generated by prost-build. +/// -------------------------------------------------------------------- +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PingRequest { + #[prost(uint64, tag = "1")] + pub version: u64, + #[prost(bytes = "vec", tag = "2")] + pub body: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct PingResponse { + #[prost(uint64, tag = "1")] + pub version: u64, + #[prost(bytes = "vec", tag = "2")] + pub body: ::prost::alloc::vec::Vec, +} +/// Generated client implementations. +pub mod node_service_client { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::http::Uri; + use tonic::codegen::*; + #[derive(Debug, Clone)] + pub struct NodeServiceClient { + inner: tonic::client::Grpc, + } + impl NodeServiceClient { + /// Attempt to create a new client by connecting to a given endpoint. + pub async fn connect(dst: D) -> Result + where + D: TryInto, + D::Error: Into, + { + let conn = tonic::transport::Endpoint::new(dst)?.connect().await?; + Ok(Self::new(conn)) + } + } + impl NodeServiceClient + where + T: tonic::client::GrpcService, + T::Error: Into, + T::ResponseBody: Body + Send + 'static, + ::Error: Into + Send, + { + pub fn new(inner: T) -> Self { + let inner = tonic::client::Grpc::new(inner); + Self { inner } + } + pub fn with_origin(inner: T, origin: Uri) -> Self { + let inner = tonic::client::Grpc::with_origin(inner, origin); + Self { inner } + } + pub fn with_interceptor(inner: T, interceptor: F) -> NodeServiceClient> + where + F: tonic::service::Interceptor, + T::ResponseBody: Default, + T: tonic::codegen::Service< + http::Request, + Response = http::Response<>::ResponseBody>, + >, + >>::Error: Into + Send + Sync, + { + NodeServiceClient::new(InterceptedService::new(inner, interceptor)) + } + /// Compress requests with the given encoding. + /// + /// This requires the server to support it otherwise it might respond with an + /// error. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.send_compressed(encoding); + self + } + /// Enable decompressing responses. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.inner = self.inner.accept_compressed(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_decoding_message_size(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.inner = self.inner.max_encoding_message_size(limit); + self + } + pub async fn ping( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Ping"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "Ping")); + self.inner.unary(req, path, codec).await + } + } +} +/// Generated server implementations. +pub mod node_service_server { + #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] + use tonic::codegen::*; + /// Generated trait containing gRPC methods that should be implemented for use with NodeServiceServer. + #[async_trait] + pub trait NodeService: Send + Sync + 'static { + async fn ping( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + } + #[derive(Debug)] + pub struct NodeServiceServer { + inner: Arc, + accept_compression_encodings: EnabledCompressionEncodings, + send_compression_encodings: EnabledCompressionEncodings, + max_decoding_message_size: Option, + max_encoding_message_size: Option, + } + impl NodeServiceServer { + pub fn new(inner: T) -> Self { + Self::from_arc(Arc::new(inner)) + } + pub fn from_arc(inner: Arc) -> Self { + Self { + inner, + accept_compression_encodings: Default::default(), + send_compression_encodings: Default::default(), + max_decoding_message_size: None, + max_encoding_message_size: None, + } + } + pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + where + F: tonic::service::Interceptor, + { + InterceptedService::new(Self::new(inner), interceptor) + } + /// Enable decompressing requests with the given encoding. + #[must_use] + pub fn accept_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.accept_compression_encodings.enable(encoding); + self + } + /// Compress responses with the given encoding, if the client supports it. + #[must_use] + pub fn send_compressed(mut self, encoding: CompressionEncoding) -> Self { + self.send_compression_encodings.enable(encoding); + self + } + /// Limits the maximum size of a decoded message. + /// + /// Default: `4MB` + #[must_use] + pub fn max_decoding_message_size(mut self, limit: usize) -> Self { + self.max_decoding_message_size = Some(limit); + self + } + /// Limits the maximum size of an encoded message. + /// + /// Default: `usize::MAX` + #[must_use] + pub fn max_encoding_message_size(mut self, limit: usize) -> Self { + self.max_encoding_message_size = Some(limit); + self + } + } + impl tonic::codegen::Service> for NodeServiceServer + where + T: NodeService, + B: Body + Send + 'static, + B::Error: Into + Send + 'static, + { + type Response = http::Response; + type Error = std::convert::Infallible; + type Future = BoxFuture; + fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + fn call(&mut self, req: http::Request) -> Self::Future { + match req.uri().path() { + "/node_service.NodeService/Ping" => { + #[allow(non_camel_case_types)] + struct PingSvc(pub Arc); + impl tonic::server::UnaryService for PingSvc { + type Response = super::PingResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::ping(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = PingSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + _ => Box::pin(async move { + Ok(http::Response::builder() + .status(200) + .header("grpc-status", tonic::Code::Unimplemented as i32) + .header(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE) + .body(empty_body()) + .unwrap()) + }), + } + } + } + impl Clone for NodeServiceServer { + fn clone(&self) -> Self { + let inner = self.inner.clone(); + Self { + inner, + accept_compression_encodings: self.accept_compression_encodings, + send_compression_encodings: self.send_compression_encodings, + max_decoding_message_size: self.max_decoding_message_size, + max_encoding_message_size: self.max_encoding_message_size, + } + } + } + impl tonic::server::NamedService for NodeServiceServer { + const NAME: &'static str = "node_service.NodeService"; + } +} diff --git a/e2e_test/Cargo.toml b/e2e_test/Cargo.toml new file mode 100644 index 00000000..a1e30a55 --- /dev/null +++ b/e2e_test/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "e2e_test" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +flatbuffers.workspace = true +protos.workspace = true +tonic = { version = "0.12.1", features = ["gzip"] } +tokio = { workspace = true } \ No newline at end of file diff --git a/e2e_test/README.md b/e2e_test/README.md new file mode 100644 index 00000000..e69de29b diff --git a/e2e_test/src/lib.rs b/e2e_test/src/lib.rs new file mode 100644 index 00000000..07abfb17 --- /dev/null +++ b/e2e_test/src/lib.rs @@ -0,0 +1 @@ +mod reliant; diff --git a/e2e_test/src/reliant/README.md b/e2e_test/src/reliant/README.md new file mode 100644 index 00000000..b1a0f384 --- /dev/null +++ b/e2e_test/src/reliant/README.md @@ -0,0 +1 @@ +The test cases in this dir need to run the cluster \ No newline at end of file diff --git a/e2e_test/src/reliant/mod.rs b/e2e_test/src/reliant/mod.rs new file mode 100644 index 00000000..67d6e260 --- /dev/null +++ b/e2e_test/src/reliant/mod.rs @@ -0,0 +1 @@ +mod node_interact_test; diff --git a/e2e_test/src/reliant/node_interact_test.rs b/e2e_test/src/reliant/node_interact_test.rs new file mode 100644 index 00000000..b2f97e13 --- /dev/null +++ b/e2e_test/src/reliant/node_interact_test.rs @@ -0,0 +1,46 @@ +#![cfg(test)] + +use protos::{ + models::{PingBody, PingBodyBuilder}, + proto_gen::node_service::{node_service_client::NodeServiceClient, PingRequest, PingResponse}, +}; +use std::error::Error; +use tonic::Request; + +#[tokio::test] +async fn main() -> Result<(), Box> { + let mut fbb = flatbuffers::FlatBufferBuilder::new(); + let payload = fbb.create_vector(b"hello world"); + + let mut builder = PingBodyBuilder::new(&mut fbb); + builder.add_payload(payload); + let root = builder.finish(); + fbb.finish(root, None); + + let finished_data = fbb.finished_data(); + + let decoded_payload = flatbuffers::root::(finished_data); + assert!(decoded_payload.is_ok()); + + // 创建客户端 + let mut client = NodeServiceClient::connect("http://localhost:9000").await?; + + // 构造 PingRequest + let request = Request::new(PingRequest { + version: 1, + body: finished_data.to_vec(), + }); + + // 发送请求并获取响应 + let response: PingResponse = client.ping(request).await?.into_inner(); + + // 打印响应 + let ping_response_body = flatbuffers::root::(&response.body); + if let Err(e) = ping_response_body { + eprintln!("{}", e); + } else { + println!("ping_resp:body(flatbuffer): {:?}", ping_response_body); + } + + Ok(()) +} diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index 5e5414f0..a155cc22 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -10,6 +10,24 @@ rust-version.workspace = true [dependencies] async-trait.workspace = true +bytes.workspace = true +clap.workspace = true +ecstore.workspace = true +flatbuffers.workspace = true +futures.workspace = true +futures-util.workspace = true +hyper.workspace = true +hyper-util.workspace = true +http.workspace = true +http-body.workspace = true +mime.workspace = true +netif.workspace = true +pin-project-lite.workspace = true +prost.workspace = true +prost-types.workspace = true +protos.workspace = true +protobuf.workspace = true +s3s.workspace = true tracing.workspace = true time = { workspace = true, features = ["parsing", "formatting"] } tokio = { workspace = true, features = [ @@ -18,22 +36,13 @@ tokio = { workspace = true, features = [ "net", "signal", ] } +tonic = { version = "0.12.1", features = ["gzip"] } +tonic-reflection.workspace = true +tower.workspace = true tracing-error.workspace = true -http.workspace = true -bytes.workspace = true -futures.workspace = true -futures-util.workspace = true +tracing-subscriber.workspace = true +transform-stream.workspace = true -ecstore = { path = "../ecstore" } -s3s = "0.10.0" -clap = { version = "4.5.7", features = ["derive"] } -tracing-subscriber = { version = "0.3.18", features = ["env-filter", "time"] } -hyper-util = { version = "0.1.5", features = [ - "tokio", - "server-auto", - "server-graceful", -] } -mime = "0.3.17" -transform-stream = "0.3.0" -netif = "0.1.6" -# pin-utils = "0.1.0" +[build-dependencies] +prost-build.workspace = true +tonic-build.workspace = true \ No newline at end of file diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs new file mode 100644 index 00000000..6f9b2ae7 --- /dev/null +++ b/rustfs/src/grpc.rs @@ -0,0 +1,47 @@ +use tonic::{Request, Response, Status}; +use tracing::{debug, error, info}; + +use protos::{ + models::{PingBody, PingBodyBuilder}, + proto_gen::node_service::{ + node_service_server::{NodeService as Node, NodeServiceServer as NodeServer}, + PingRequest, PingResponse, + }, +}; + +#[derive(Debug)] +struct NodeService {} + +pub fn make_server() -> NodeServer { + NodeServer::new(NodeService {}) +} + +#[tonic::async_trait] +impl Node for NodeService { + async fn ping(&self, request: Request) -> Result, Status> { + debug!("PING"); + + let ping_req = request.into_inner(); + let ping_body = flatbuffers::root::(&ping_req.body); + if let Err(e) = ping_body { + error!("{}", e); + } else { + info!("ping_req:body(flatbuffer): {:?}", ping_body); + } + + let mut fbb = flatbuffers::FlatBufferBuilder::new(); + let payload = fbb.create_vector(b"hello, caller"); + + let mut builder = PingBodyBuilder::new(&mut fbb); + builder.add_payload(payload); + let root = builder.finish(); + fbb.finish(root, None); + + let finished_data = fbb.finished_data(); + + Ok(tonic::Response::new(PingResponse { + version: 1, + body: finished_data.to_vec(), + })) + } +} diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index d43b4ab5..113f81a7 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -1,13 +1,18 @@ mod config; +mod grpc; +mod service; mod storage; use clap::Parser; use ecstore::error::Result; +use grpc::make_server; use hyper_util::{ rt::{TokioExecutor, TokioIo}, server::conn::auto::Builder as ConnBuilder, + service::TowerToHyperService, }; use s3s::{auth::SimpleAuth, service::S3ServiceBuilder}; +use service::hybrid; use std::{io::IsTerminal, net::SocketAddr, str::FromStr}; use tokio::net::TcpListener; use tracing::{debug, info}; @@ -96,6 +101,8 @@ async fn run(opt: config::Opt) -> Result<()> { let hyper_service = service.into_shared(); + let hybrid_service = TowerToHyperService::new(hybrid(hyper_service, make_server())); + let http_server = ConnBuilder::new(TokioExecutor::new()); let graceful = hyper_util::server::graceful::GracefulShutdown::new(); @@ -119,7 +126,7 @@ async fn run(opt: config::Opt) -> Result<()> { } }; - let conn = http_server.serve_connection(TokioIo::new(socket), hyper_service.clone()); + let conn = http_server.serve_connection(TokioIo::new(socket), hybrid_service.clone()); let conn = graceful.watch(conn.into_owned()); tokio::spawn(async move { let _ = conn.await; diff --git a/rustfs/src/service.rs b/rustfs/src/service.rs new file mode 100644 index 00000000..ea1d4272 --- /dev/null +++ b/rustfs/src/service.rs @@ -0,0 +1,150 @@ +use std::pin::Pin; +use std::task::{Context, Poll}; + +use futures::Future; +use http_body::Frame; +use hyper::body::Incoming; +use hyper::{Request, Response}; +use pin_project_lite::pin_project; +use tower::Service; + +type BoxError = Box; + +/// Generate a [`HybridService`] +pub(crate) fn hybrid(make_rest: MakeRest, grpc: Grpc) -> HybridService { + HybridService { rest: make_rest, grpc } +} + +/// The service that can serve both gRPC and REST HTTP Requests +#[derive(Clone)] +pub struct HybridService { + rest: Rest, + grpc: Grpc, +} + +impl Service> for HybridService +where + Rest: Service, Response = Response>, + Grpc: Service, Response = Response>, + Rest::Error: Into, + Grpc::Error: Into, +{ + type Response = Response>; + type Error = BoxError; + type Future = HybridFuture; + + fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll> { + match self.rest.poll_ready(cx) { + Poll::Ready(Ok(())) => match self.grpc.poll_ready(cx) { + Poll::Ready(Ok(())) => Poll::Ready(Ok(())), + Poll::Ready(Err(e)) => Poll::Ready(Err(e.into())), + Poll::Pending => Poll::Pending, + }, + Poll::Ready(Err(e)) => Poll::Ready(Err(e.into())), + Poll::Pending => Poll::Pending, + } + } + + /// When calling the service, gRPC is served if the HTTP request version is HTTP/2 + /// and if the Content-Type is "application/grpc"; otherwise, the request is served + /// as a REST request + fn call(&mut self, req: Request) -> Self::Future { + match (req.version(), req.headers().get(hyper::header::CONTENT_TYPE)) { + (hyper::Version::HTTP_2, Some(hv)) if hv.as_bytes().starts_with(b"application/grpc") => HybridFuture::Grpc { + grpc_future: self.grpc.call(req), + }, + _ => HybridFuture::Rest { + rest_future: self.rest.call(req), + }, + } + } +} + +pin_project! { + /// A hybrid HTTP body that will be used in the response type for the + /// [`HybridFuture`], i.e., the output of the [`HybridService`] + #[project = HybridBodyProj] + pub enum HybridBody { + Rest { + #[pin] + rest_body: RestBody + }, + Grpc { + #[pin] + grpc_body: GrpcBody + }, + } +} + +impl http_body::Body for HybridBody +where + RestBody: http_body::Body + Send + Unpin, + GrpcBody: http_body::Body + Send + Unpin, + RestBody::Error: Into, + GrpcBody::Error: Into, +{ + type Data = RestBody::Data; + type Error = BoxError; + + fn is_end_stream(&self) -> bool { + match self { + Self::Rest { rest_body } => rest_body.is_end_stream(), + Self::Grpc { grpc_body } => grpc_body.is_end_stream(), + } + } + + fn poll_frame(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll, Self::Error>>> { + match self.project() { + HybridBodyProj::Rest { rest_body } => rest_body.poll_frame(cx).map_err(Into::into), + HybridBodyProj::Grpc { grpc_body } => grpc_body.poll_frame(cx).map_err(Into::into), + } + } + + fn size_hint(&self) -> http_body::SizeHint { + match self { + Self::Rest { rest_body } => rest_body.size_hint(), + Self::Grpc { grpc_body } => grpc_body.size_hint(), + } + } +} + +pin_project! { + /// A future that accepts an HTTP request as input and returns an HTTP + /// response as output for the [`HybridService`] + #[project = HybridFutureProj] + pub enum HybridFuture { + Rest { + #[pin] + rest_future: RestFuture, + }, + Grpc { + #[pin] + grpc_future: GrpcFuture, + } + } +} + +impl Future for HybridFuture +where + RestFuture: Future, RestError>>, + GrpcFuture: Future, GrpcError>>, + RestError: Into, + GrpcError: Into, +{ + type Output = Result>, BoxError>; + + fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { + match self.project() { + HybridFutureProj::Rest { rest_future } => match rest_future.poll(cx) { + Poll::Ready(Ok(res)) => Poll::Ready(Ok(res.map(|rest_body| HybridBody::Rest { rest_body }))), + Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())), + Poll::Pending => Poll::Pending, + }, + HybridFutureProj::Grpc { grpc_future } => match grpc_future.poll(cx) { + Poll::Ready(Ok(res)) => Poll::Ready(Ok(res.map(|grpc_body| HybridBody::Grpc { grpc_body }))), + Poll::Ready(Err(err)) => Poll::Ready(Err(err.into())), + Poll::Pending => Poll::Pending, + }, + } + } +} From 78f3944b5d9ce03331eff3bff0b4e0eb8b98eb4f Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Tue, 27 Aug 2024 16:43:12 +0800 Subject: [PATCH 04/23] remote make bucket Signed-off-by: junxiang Mu <1948535941@qq.com> --- Cargo.lock | 3 + Cargo.toml | 2 +- common/protos/build.rs | 12 ++-- .../flatbuffers_generated/mod.rs | 0 .../flatbuffers_generated/models.rs | 0 common/protos/src/generated/mod.rs | 6 ++ .../src/{ => generated}/proto_gen/mod.rs | 0 .../{ => generated}/proto_gen/node_service.rs | 69 +++++++++++++++++++ common/protos/src/lib.rs | 32 +++++++-- common/protos/src/node.proto | 15 ++++ coordinator/Cargo.toml | 15 ++++ coordinator/src/lib.rs | 5 ++ ecstore/Cargo.toml | 3 + ecstore/src/peer.rs | 61 +++++++++++----- rustfs/src/grpc.rs | 10 ++- 15 files changed, 206 insertions(+), 27 deletions(-) rename common/protos/src/{ => generated}/flatbuffers_generated/mod.rs (100%) rename common/protos/src/{ => generated}/flatbuffers_generated/models.rs (100%) create mode 100644 common/protos/src/generated/mod.rs rename common/protos/src/{ => generated}/proto_gen/mod.rs (100%) rename common/protos/src/{ => generated}/proto_gen/node_service.rs (74%) create mode 100644 coordinator/Cargo.toml create mode 100644 coordinator/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index ce909a40..f283234a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -430,6 +430,7 @@ dependencies = [ "openssl", "path-absolutize", "path-clean", + "protos", "reed-solomon-erasure", "regex", "rmp", @@ -444,6 +445,8 @@ dependencies = [ "tokio", "tokio-stream", "tokio-util", + "tonic", + "tower", "tracing", "tracing-error", "transform-stream", diff --git a/Cargo.toml b/Cargo.toml index 0ec2e78d..ca8430b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -49,7 +49,7 @@ tokio = { version = "1.38.0", features = ["fs", "rt-multi-thread"] } tonic = { version = "0.12.1", features = ["gzip"] } tonic-build = "0.12.1" tonic-reflection = "0.12" -tower = "0.4.13" +tower = { version = "0.4.13", features = ["timeout"] } tracing = "0.1.40" tracing-error = "0.2.0" tracing-subscriber = { version = "0.3.18", features = ["env-filter", "time"] } diff --git a/common/protos/build.rs b/common/protos/build.rs index 35efacb7..5ef32dde 100644 --- a/common/protos/build.rs +++ b/common/protos/build.rs @@ -40,8 +40,8 @@ fn main() -> Result<(), AnyError> { let project_root_dir = env::current_dir()?; let proto_dir = project_root_dir.join("src"); let proto_files = &["node.proto"]; - let proto_out_dir = project_root_dir.join("src").join("proto_gen"); - let flatbuffer_out_dir = project_root_dir.join("src").join("flatbuffers_generated"); + let proto_out_dir = project_root_dir.join("src").join("generated").join("proto_gen"); + let flatbuffer_out_dir = project_root_dir.join("src").join("generated").join("flatbuffers_generated"); let descriptor_set_path = PathBuf::from(env::var(ENV_OUT_DIR).unwrap()).join("proto-descriptor.bin"); tonic_build::configure() @@ -54,13 +54,17 @@ fn main() -> Result<(), AnyError> { .map_err(|e| format!("Failed to generate protobuf file: {e}."))?; // protos/gen/mod.rs - let generated_mod_rs_path = project_root_dir.join("src").join("proto_gen").join("mod.rs"); + let generated_mod_rs_path = project_root_dir + .join("src") + .join("generated") + .join("proto_gen") + .join("mod.rs"); let mut generated_mod_rs = fs::File::create(generated_mod_rs_path)?; writeln!(&mut generated_mod_rs, "pub mod node_service;")?; generated_mod_rs.flush()?; - let generated_mod_rs_path = project_root_dir.join("src").join("lib.rs"); + let generated_mod_rs_path = project_root_dir.join("src").join("generated").join("mod.rs"); let mut generated_mod_rs = fs::File::create(generated_mod_rs_path)?; writeln!(&mut generated_mod_rs, "#![allow(unused_imports)]")?; diff --git a/common/protos/src/flatbuffers_generated/mod.rs b/common/protos/src/generated/flatbuffers_generated/mod.rs similarity index 100% rename from common/protos/src/flatbuffers_generated/mod.rs rename to common/protos/src/generated/flatbuffers_generated/mod.rs diff --git a/common/protos/src/flatbuffers_generated/models.rs b/common/protos/src/generated/flatbuffers_generated/models.rs similarity index 100% rename from common/protos/src/flatbuffers_generated/models.rs rename to common/protos/src/generated/flatbuffers_generated/models.rs diff --git a/common/protos/src/generated/mod.rs b/common/protos/src/generated/mod.rs new file mode 100644 index 00000000..4ab5a438 --- /dev/null +++ b/common/protos/src/generated/mod.rs @@ -0,0 +1,6 @@ +#![allow(unused_imports)] +#![allow(clippy::all)] +pub mod proto_gen; + +mod flatbuffers_generated; +pub use flatbuffers_generated::models::*; diff --git a/common/protos/src/proto_gen/mod.rs b/common/protos/src/generated/proto_gen/mod.rs similarity index 100% rename from common/protos/src/proto_gen/mod.rs rename to common/protos/src/generated/proto_gen/mod.rs diff --git a/common/protos/src/proto_gen/node_service.rs b/common/protos/src/generated/proto_gen/node_service.rs similarity index 74% rename from common/protos/src/proto_gen/node_service.rs rename to common/protos/src/generated/proto_gen/node_service.rs index 463aac82..55b5b01a 100644 --- a/common/protos/src/proto_gen/node_service.rs +++ b/common/protos/src/generated/proto_gen/node_service.rs @@ -16,6 +16,28 @@ pub struct PingResponse { #[prost(bytes = "vec", tag = "2")] pub body: ::prost::alloc::vec::Vec, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, Copy, PartialEq, ::prost::Message)] +pub struct MakeBucketOptions { + #[prost(bool, tag = "1")] + pub force_create: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MakeBucketRequest { + #[prost(string, tag = "1")] + pub name: ::prost::alloc::string::String, + #[prost(message, optional, tag = "2")] + pub options: ::core::option::Option, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MakeBucketResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} /// Generated client implementations. pub mod node_service_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] @@ -109,6 +131,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "Ping")); self.inner.unary(req, path, codec).await } + pub async fn make_bucket( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeBucket"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "MakeBucket")); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -122,6 +159,10 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn make_bucket( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] pub struct NodeServiceServer { @@ -221,6 +262,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/MakeBucket" => { + #[allow(non_camel_case_types)] + struct MakeBucketSvc(pub Arc); + impl tonic::server::UnaryService for MakeBucketSvc { + type Response = super::MakeBucketResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::make_bucket(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = MakeBucketSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } _ => Box::pin(async move { Ok(http::Response::builder() .status(200) diff --git a/common/protos/src/lib.rs b/common/protos/src/lib.rs index 4ab5a438..639f9b06 100644 --- a/common/protos/src/lib.rs +++ b/common/protos/src/lib.rs @@ -1,6 +1,28 @@ -#![allow(unused_imports)] -#![allow(clippy::all)] -pub mod proto_gen; +mod generated; +use std::time::Duration; -mod flatbuffers_generated; -pub use flatbuffers_generated::models::*; +pub use generated::*; +use proto_gen::node_service::node_service_client::NodeServiceClient; +use tonic::{codec::CompressionEncoding, transport::Channel}; +use tower::timeout::Timeout; + +// Default 100 MB +pub const DEFAULT_GRPC_SERVER_MESSAGE_LEN: usize = 100 * 1024 * 1024; + +pub fn node_service_time_out_client( + channel: Channel, + time_out: Duration, + max_message_size: usize, + grpc_enable_gzip: bool, +) -> NodeServiceClient> { + let timeout_channel = Timeout::new(channel, time_out); + let client = NodeServiceClient::>::new(timeout_channel); + let client = NodeServiceClient::max_decoding_message_size(client, max_message_size); + if grpc_enable_gzip { + NodeServiceClient::max_encoding_message_size(client, max_message_size) + .accept_compressed(CompressionEncoding::Gzip) + .send_compressed(CompressionEncoding::Gzip) + } else { + NodeServiceClient::max_encoding_message_size(client, max_message_size) + } +} diff --git a/common/protos/src/node.proto b/common/protos/src/node.proto index 879eb97f..c3d6cd29 100644 --- a/common/protos/src/node.proto +++ b/common/protos/src/node.proto @@ -12,8 +12,23 @@ message PingResponse { bytes body = 2; } +message MakeBucketOptions { + bool force_create = 1; +} + +message MakeBucketRequest { + string name = 1; + MakeBucketOptions options = 2; +} + +message MakeBucketResponse { + bool success = 1; + optional string error_info = 2; +} + /* -------------------------------------------------------------------- */ service NodeService { rpc Ping(PingRequest) returns (PingResponse) {}; + rpc MakeBucket(MakeBucketRequest) returns (MakeBucketResponse) {}; } \ No newline at end of file diff --git a/coordinator/Cargo.toml b/coordinator/Cargo.toml new file mode 100644 index 00000000..a1e30a55 --- /dev/null +++ b/coordinator/Cargo.toml @@ -0,0 +1,15 @@ +[package] +name = "e2e_test" +version.workspace = true +edition.workspace = true +license.workspace = true +repository.workspace = true +rust-version.workspace = true + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] +flatbuffers.workspace = true +protos.workspace = true +tonic = { version = "0.12.1", features = ["gzip"] } +tokio = { workspace = true } \ No newline at end of file diff --git a/coordinator/src/lib.rs b/coordinator/src/lib.rs new file mode 100644 index 00000000..10439ca7 --- /dev/null +++ b/coordinator/src/lib.rs @@ -0,0 +1,5 @@ +#[async_trait::async_trait] +pub trait Coordinator: Send + Sync { + async fn ping(); + async fn create_bucket() -> Result<> +} \ No newline at end of file diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index ac8e1948..de373425 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -28,6 +28,7 @@ lazy_static = "1.5.0" regex = "1.10.5" netif = "0.1.6" path-absolutize = "3.1.1" +protos.workspace = true rmp-serde = "1.3.0" tokio-util = { version = "0.7.11", features = ["io"] } s3s = "0.10.0" @@ -38,6 +39,8 @@ sha2 = "0.10.8" hex-simd = "0.8.0" path-clean = "1.0.1" tokio-stream = "0.1.15" +tonic.workspace = true +tower.workspace = true rmp = "0.8.14" byteorder = "1.5.0" xxhash-rust = { version = "0.8.12", features = ["xxh64"] } diff --git a/ecstore/src/peer.rs b/ecstore/src/peer.rs index 7329923c..0c9dd58e 100644 --- a/ecstore/src/peer.rs +++ b/ecstore/src/peer.rs @@ -1,7 +1,14 @@ use async_trait::async_trait; use futures::future::join_all; +use protos::proto_gen::node_service::MakeBucketRequest; +use protos::{ + node_service_time_out_client, proto_gen::node_service::MakeBucketOptions as proto_MakeBucketOptions, + DEFAULT_GRPC_SERVER_MESSAGE_LEN, +}; use regex::Regex; -use std::{collections::HashMap, fmt::Debug, sync::Arc}; +use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration}; +use tonic::transport::{Channel, Endpoint}; +use tonic::Request; use tracing::warn; use crate::{ @@ -56,9 +63,8 @@ impl S3PeerSys { } } -#[async_trait] -impl PeerS3Client for S3PeerSys { - async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { +impl S3PeerSys { + pub async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { let mut futures = Vec::with_capacity(self.clients.len()); for cli in self.clients.iter() { futures.push(cli.make_bucket(bucket, opts)); @@ -95,7 +101,7 @@ impl PeerS3Client for S3PeerSys { Ok(()) } - async fn list_bucket(&self, opts: &BucketOptions) -> Result> { + pub async fn list_bucket(&self, opts: &BucketOptions) -> Result> { let mut futures = Vec::with_capacity(self.clients.len()); for cli in self.clients.iter() { futures.push(cli.list_bucket(opts)); @@ -140,7 +146,7 @@ impl PeerS3Client for S3PeerSys { Ok(buckets) } - async fn delete_bucket(&self, bucket: &str) -> Result<()> { + pub async fn delete_bucket(&self, bucket: &str) -> Result<()> { let mut futures = Vec::with_capacity(self.clients.len()); for cli in self.clients.iter() { futures.push(cli.delete_bucket(bucket)); @@ -165,7 +171,7 @@ impl PeerS3Client for S3PeerSys { Ok(()) } - async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result { + pub async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result { let mut futures = Vec::with_capacity(self.clients.len()); for cli in self.clients.iter() { futures.push(cli.get_bucket_info(bucket, opts)); @@ -206,7 +212,7 @@ impl PeerS3Client for S3PeerSys { .ok_or(Error::new(DiskError::VolumeNotFound)) } - fn get_pools(&self) -> Vec { + pub fn get_pools(&self) -> Vec { unimplemented!() } } @@ -378,27 +384,50 @@ impl PeerS3Client for LocalPeerS3Client { #[derive(Debug)] pub struct RemotePeerS3Client { - // pub node: Node, - // pub pools: Vec, + pub _node: Node, + pub pools: Vec, + pub channel: Channel, } impl RemotePeerS3Client { - fn new(_node: Node, _pools: Vec) -> Self { - // Self { node, pools } - Self {} + fn new(node: Node, pools: Vec) -> Self { + let connector = Endpoint::from_shared(format!("{}", node.url.clone().as_str())).unwrap(); + let channel = tokio::runtime::Runtime::new().unwrap().block_on(connector.connect()).unwrap(); + Self { + _node: node, + pools, + channel, + } } } #[async_trait] impl PeerS3Client for RemotePeerS3Client { fn get_pools(&self) -> Vec { - unimplemented!() + self.pools.clone() } async fn list_bucket(&self, _opts: &BucketOptions) -> Result> { unimplemented!() } - async fn make_bucket(&self, _bucket: &str, _opts: &MakeBucketOptions) -> Result<()> { - unimplemented!() + async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { + let mut client = node_service_time_out_client( + self.channel.clone(), + Duration::new(30, 0), // TODO: use config setting + DEFAULT_GRPC_SERVER_MESSAGE_LEN, + // grpc_enable_gzip, + false, // TODO: use config setting + ); + let request = Request::new(MakeBucketRequest { + name: bucket.to_string(), + options: Some(proto_MakeBucketOptions { + force_create: opts.force_create, + }), + }); + let _response = client.make_bucket(request).await?.into_inner(); + + // TODO: deal with error + + Ok(()) } async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> Result { unimplemented!() diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 6f9b2ae7..63403b7e 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -5,7 +5,7 @@ use protos::{ models::{PingBody, PingBodyBuilder}, proto_gen::node_service::{ node_service_server::{NodeService as Node, NodeServiceServer as NodeServer}, - PingRequest, PingResponse, + MakeBucketRequest, MakeBucketResponse, PingRequest, PingResponse, }, }; @@ -44,4 +44,12 @@ impl Node for NodeService { body: finished_data.to_vec(), })) } + + async fn make_bucket(&self, request: Request) -> Result, Status> { + debug!("make bucket"); + + let req = request.into_inner(); + + unimplemented!() + } } From 38fafed0133413c59cf23495c10955167dcf6917 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 27 Aug 2024 17:15:47 +0800 Subject: [PATCH 05/23] rpc add local_peer --- ecstore/src/lib.rs | 2 +- ecstore/src/peer.rs | 29 +++++++++++++++-------------- ecstore/src/store.rs | 6 +++++- ecstore/src/store_api.rs | 1 + rustfs/src/grpc.rs | 13 +++++++++---- rustfs/src/main.rs | 9 ++++++--- rustfs/src/storage/ecfs.rs | 3 +-- 7 files changed, 38 insertions(+), 25 deletions(-) diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 1371d1f1..82741b44 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -6,7 +6,7 @@ mod endpoints; mod erasure; pub mod error; mod file_meta; -mod peer; +pub mod peer; pub mod set_disk; mod sets; pub mod store; diff --git a/ecstore/src/peer.rs b/ecstore/src/peer.rs index 0c9dd58e..4274c297 100644 --- a/ecstore/src/peer.rs +++ b/ecstore/src/peer.rs @@ -26,7 +26,7 @@ pub trait PeerS3Client: Debug + Sync + Send + 'static { async fn list_bucket(&self, opts: &BucketOptions) -> Result>; async fn delete_bucket(&self, bucket: &str) -> Result<()>; async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result; - fn get_pools(&self) -> Vec; + fn get_pools(&self) -> Option>; } #[derive(Debug)] @@ -50,10 +50,10 @@ impl S3PeerSys { .map(|e| { if e.is_local { let cli: Box = - Box::new(LocalPeerS3Client::new(local_disks.clone(), e.clone(), e.pools.clone())); + Box::new(LocalPeerS3Client::new(local_disks.clone(), Some(e.clone()), Some(e.pools.clone()))); Arc::new(cli) } else { - let cli: Box = Box::new(RemotePeerS3Client::new(e.clone(), e.pools.clone())); + let cli: Box = Box::new(RemotePeerS3Client::new(Some(e.clone()), Some(e.pools.clone()))); Arc::new(cli) } }) @@ -89,7 +89,7 @@ impl S3PeerSys { for (j, cli) in self.clients.iter().enumerate() { let pools = cli.get_pools(); let idx = i; - if pools.contains(&idx) { + if pools.unwrap_or_default().contains(&idx) { per_pool_errs.push(errors[j].as_ref()); } @@ -199,7 +199,7 @@ impl S3PeerSys { for (j, cli) in self.clients.iter().enumerate() { let pools = cli.get_pools(); let idx = i; - if pools.contains(&idx) { + if pools.unwrap_or_default().contains(&idx) { per_pool_errs.push(errors[j].as_ref()); } @@ -212,7 +212,7 @@ impl S3PeerSys { .ok_or(Error::new(DiskError::VolumeNotFound)) } - pub fn get_pools(&self) -> Vec { + pub fn get_pools(&self) -> Option> { unimplemented!() } } @@ -221,11 +221,11 @@ impl S3PeerSys { pub struct LocalPeerS3Client { pub local_disks: Vec, // pub node: Node, - pub pools: Vec, + pub pools: Option>, } impl LocalPeerS3Client { - fn new(local_disks: Vec, _node: Node, pools: Vec) -> Self { + pub fn new(local_disks: Vec, _node: Option, pools: Option>) -> Self { Self { local_disks, // node, @@ -236,7 +236,7 @@ impl LocalPeerS3Client { #[async_trait] impl PeerS3Client for LocalPeerS3Client { - fn get_pools(&self) -> Vec { + fn get_pools(&self) -> Option> { self.pools.clone() } async fn list_bucket(&self, _opts: &BucketOptions) -> Result> { @@ -384,14 +384,15 @@ impl PeerS3Client for LocalPeerS3Client { #[derive(Debug)] pub struct RemotePeerS3Client { - pub _node: Node, - pub pools: Vec, + pub _node: Option, + pub pools: Option>, pub channel: Channel, } impl RemotePeerS3Client { - fn new(node: Node, pools: Vec) -> Self { - let connector = Endpoint::from_shared(format!("{}", node.url.clone().as_str())).unwrap(); + fn new(node: Option, pools: Option>) -> Self { + let connector = + Endpoint::from_shared(format!("{}", node.as_ref().map(|v| { v.url.to_string() }).unwrap_or_default())).unwrap(); let channel = tokio::runtime::Runtime::new().unwrap().block_on(connector.connect()).unwrap(); Self { _node: node, @@ -403,7 +404,7 @@ impl RemotePeerS3Client { #[async_trait] impl PeerS3Client for RemotePeerS3Client { - fn get_pools(&self) -> Vec { + fn get_pools(&self) -> Option> { self.pools.clone() } async fn list_bucket(&self, _opts: &BucketOptions) -> Result> { diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index bbcc49fa..b31a5fc4 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -4,7 +4,7 @@ use crate::{ disks_layout::DisksLayout, endpoints::EndpointServerPools, error::{Error, Result}, - peer::{PeerS3Client, S3PeerSys}, + peer::S3PeerSys, sets::Sets, store_api::{ BucketInfo, BucketOptions, CompletePart, GetObjectReader, HTTPRangeSpec, ListObjectsInfo, ListObjectsV2Info, @@ -104,6 +104,10 @@ impl ECStore { }) } + pub fn local_disks(&self) -> Vec { + self.local_disks.clone() + } + fn single_pool(&self) -> bool { self.pools.len() == 1 } diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index 3aa50f56..a1556b10 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -242,6 +242,7 @@ pub enum BitrotAlgorithm { BLAKE2b512, } +#[derive(Debug, Default)] pub struct MakeBucketOptions { pub force_create: bool, } diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 63403b7e..890e6cba 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -1,3 +1,4 @@ +use ecstore::{disk::DiskStore, peer::LocalPeerS3Client}; use tonic::{Request, Response, Status}; use tracing::{debug, error, info}; @@ -10,10 +11,13 @@ use protos::{ }; #[derive(Debug)] -struct NodeService {} +struct NodeService { + pub local_peer: LocalPeerS3Client, +} -pub fn make_server() -> NodeServer { - NodeServer::new(NodeService {}) +pub fn make_server(local_disks: Vec) -> NodeServer { + let local_peer = LocalPeerS3Client::new(local_disks, None, None); + NodeServer::new(NodeService { local_peer }) } #[tonic::async_trait] @@ -48,8 +52,9 @@ impl Node for NodeService { async fn make_bucket(&self, request: Request) -> Result, Status> { debug!("make bucket"); - let req = request.into_inner(); + let _req = request.into_inner(); + // match self.local_peer.make_bucket(&name, &MakeBucketOptions::default()).await {} unimplemented!() } } diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 113f81a7..c0567d5e 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -4,7 +4,7 @@ mod service; mod storage; use clap::Parser; -use ecstore::error::Result; +use ecstore::{error::Result, store::ECStore}; use grpc::make_server; use hyper_util::{ rt::{TokioExecutor, TokioIo}, @@ -66,9 +66,12 @@ async fn run(opt: config::Opt) -> Result<()> { // }) // }; + let store: ECStore = ECStore::new(opt.address.clone(), opt.volumes.clone()).await?; + let local_disks = store.local_disks(); + // Setup S3 service let service = { - let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), opt.volumes.clone()).await?); + let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(store).await?); let mut access_key = String::from_str(config::DEFAULT_ACCESS_KEY).unwrap(); let mut secret_key = String::from_str(config::DEFAULT_SECRET_KEY).unwrap(); @@ -101,7 +104,7 @@ async fn run(opt: config::Opt) -> Result<()> { let hyper_service = service.into_shared(); - let hybrid_service = TowerToHyperService::new(hybrid(hyper_service, make_server())); + let hybrid_service = TowerToHyperService::new(hybrid(hyper_service, make_server(local_disks))); let http_server = ConnBuilder::new(TokioExecutor::new()); let graceful = hyper_util::server::graceful::GracefulShutdown::new(); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 8e8f53db..76b091ac 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -43,8 +43,7 @@ pub struct FS { } impl FS { - pub async fn new(address: String, endpoints: Vec) -> Result { - let store: ECStore = ECStore::new(address, endpoints).await?; + pub async fn new(store: ECStore) -> Result { Ok(Self { store }) } } From a8e546c9af4163660fd7d948a40dbbf37cbfd1fa Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Thu, 29 Aug 2024 18:28:47 +0800 Subject: [PATCH 06/23] support remote disk Signed-off-by: junxiang Mu <1948535941@qq.com> --- Cargo.lock | 3 + .../src/generated/proto_gen/node_service.rs | 1221 +++++++++++++++++ common/protos/src/node.proto | 252 +++- e2e_test/Cargo.toml | 2 + e2e_test/src/reliant/node_interact_test.rs | 46 +- ecstore/src/disk/local.rs | 18 +- ecstore/src/disk/mod.rs | 227 ++- ecstore/src/disk/remote.rs | 356 +++++ ecstore/src/erasure.rs | 17 +- ecstore/src/lib.rs | 2 +- ecstore/src/store_api.rs | 1 + rustfs/Cargo.toml | 1 + rustfs/src/grpc.rs | 609 +++++++- 13 files changed, 2688 insertions(+), 67 deletions(-) create mode 100644 ecstore/src/disk/remote.rs diff --git a/Cargo.lock b/Cargo.lock index f283234a..df15b64e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -407,8 +407,10 @@ dependencies = [ name = "e2e_test" version = "0.0.1" dependencies = [ + "ecstore", "flatbuffers", "protos", + "serde_json", "tokio", "tonic", ] @@ -1550,6 +1552,7 @@ dependencies = [ "protobuf", "protos", "s3s", + "serde_json", "time", "tokio", "tonic", diff --git a/common/protos/src/generated/proto_gen/node_service.rs b/common/protos/src/generated/proto_gen/node_service.rs index 55b5b01a..13e968bf 100644 --- a/common/protos/src/generated/proto_gen/node_service.rs +++ b/common/protos/src/generated/proto_gen/node_service.rs @@ -38,6 +38,377 @@ pub struct MakeBucketResponse { #[prost(string, optional, tag = "2")] pub error_info: ::core::option::Option<::prost::alloc::string::String>, } +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadAllRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadAllResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(bytes = "vec", tag = "2")] + pub data: ::prost::alloc::vec::Vec, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WriteAllRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(bytes = "vec", tag = "4")] + pub data: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WriteAllResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub options: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RenameFileRequst { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub src_volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub src_path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub dst_volume: ::prost::alloc::string::String, + #[prost(string, tag = "5")] + pub dst_path: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RenameFileResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WriteRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(bool, tag = "4")] + pub is_append: bool, + #[prost(bytes = "vec", tag = "5")] + pub data: ::prost::alloc::vec::Vec, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WriteResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadAtRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(int64, tag = "4")] + pub offset: i64, + #[prost(int64, tag = "5")] + pub length: i64, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadAtResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(bytes = "vec", tag = "2")] + pub data: ::prost::alloc::vec::Vec, + #[prost(int64, tag = "3")] + pub read_size: i64, + #[prost(string, optional, tag = "4")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListDirRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListDirResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, repeated, tag = "2")] + pub volumes: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WalkDirRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub walk_dir_options: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WalkDirResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, repeated, tag = "2")] + pub meta_cache_entry: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RenameDataRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub src_volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub src_path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub file_info: ::prost::alloc::string::String, + #[prost(string, tag = "5")] + pub dst_volume: ::prost::alloc::string::String, + #[prost(string, tag = "6")] + pub dst_path: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct RenameDataResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub rename_data_resp: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MakeVolumesRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, repeated, tag = "2")] + pub volumes: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MakeVolumesResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MakeVolumeRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct MakeVolumeResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListVolumesRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListVolumesResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, repeated, tag = "2")] + pub volume_infos: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StatVolumeRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct StatVolumeResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub volume_info: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WriteMetadataRequest { + /// indicate which one in the disks + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub file_info: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct WriteMetadataResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadVersionRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(string, tag = "4")] + pub version_id: ::prost::alloc::string::String, + #[prost(string, tag = "5")] + pub opts: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadVersionResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub file_info: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadXlRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub path: ::prost::alloc::string::String, + #[prost(bool, tag = "4")] + pub read_data: bool, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadXlResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub raw_file_info: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadMultipleRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub read_multiple_req: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ReadMultipleResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, repeated, tag = "2")] + pub read_multiple_resps: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteVolumeRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteVolumeResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} /// Generated client implementations. pub mod node_service_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] @@ -116,6 +487,7 @@ pub mod node_service_client { self.inner = self.inner.max_encoding_message_size(limit); self } + /// -------------------------------meta service-------------------------- pub async fn ping( &mut self, request: impl tonic::IntoRequest, @@ -146,6 +518,277 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "MakeBucket")); self.inner.unary(req, path, codec).await } + pub async fn read_all( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadAll"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ReadAll")); + self.inner.unary(req, path, codec).await + } + pub async fn write_all( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteAll"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "WriteAll")); + self.inner.unary(req, path, codec).await + } + pub async fn delete( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Delete"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "Delete")); + self.inner.unary(req, path, codec).await + } + pub async fn rename_file( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameFile"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "RenameFile")); + self.inner.unary(req, path, codec).await + } + pub async fn write( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Write"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "Write")); + self.inner.unary(req, path, codec).await + } + /// rpc Append(AppendRequest) returns (AppendResponse) {}; + pub async fn read_at( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadAt"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ReadAt")); + self.inner.unary(req, path, codec).await + } + pub async fn list_dir( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListDir"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ListDir")); + self.inner.unary(req, path, codec).await + } + pub async fn walk_dir( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WalkDir"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "WalkDir")); + self.inner.unary(req, path, codec).await + } + pub async fn rename_data( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameData"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "RenameData")); + self.inner.unary(req, path, codec).await + } + pub async fn make_volumes( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeVolumes"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "MakeVolumes")); + self.inner.unary(req, path, codec).await + } + pub async fn make_volume( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeVolume"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "MakeVolume")); + self.inner.unary(req, path, codec).await + } + pub async fn list_volumes( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListVolumes"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ListVolumes")); + self.inner.unary(req, path, codec).await + } + pub async fn stat_volume( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/StatVolume"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "StatVolume")); + self.inner.unary(req, path, codec).await + } + pub async fn write_metadata( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteMetadata"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "WriteMetadata")); + self.inner.unary(req, path, codec).await + } + pub async fn read_version( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadVersion"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ReadVersion")); + self.inner.unary(req, path, codec).await + } + pub async fn read_xl( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadXL"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ReadXL")); + self.inner.unary(req, path, codec).await + } + pub async fn read_multiple( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadMultiple"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ReadMultiple")); + self.inner.unary(req, path, codec).await + } + pub async fn delete_volume( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVolume"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "DeleteVolume")); + self.inner.unary(req, path, codec).await + } } } /// Generated server implementations. @@ -155,6 +798,7 @@ pub mod node_service_server { /// Generated trait containing gRPC methods that should be implemented for use with NodeServiceServer. #[async_trait] pub trait NodeService: Send + Sync + 'static { + /// -------------------------------meta service-------------------------- async fn ping( &self, request: tonic::Request, @@ -163,6 +807,79 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn read_all( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn write_all( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn delete( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn rename_file( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn write( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + /// rpc Append(AppendRequest) returns (AppendResponse) {}; + async fn read_at( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn list_dir( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn walk_dir( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn rename_data( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn make_volumes( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn make_volume( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn list_volumes( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn stat_volume( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn write_metadata( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn read_version( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn read_xl( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn read_multiple( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn delete_volume( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; } #[derive(Debug)] pub struct NodeServiceServer { @@ -290,6 +1007,510 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/ReadAll" => { + #[allow(non_camel_case_types)] + struct ReadAllSvc(pub Arc); + impl tonic::server::UnaryService for ReadAllSvc { + type Response = super::ReadAllResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::read_all(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ReadAllSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/WriteAll" => { + #[allow(non_camel_case_types)] + struct WriteAllSvc(pub Arc); + impl tonic::server::UnaryService for WriteAllSvc { + type Response = super::WriteAllResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::write_all(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = WriteAllSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/Delete" => { + #[allow(non_camel_case_types)] + struct DeleteSvc(pub Arc); + impl tonic::server::UnaryService for DeleteSvc { + type Response = super::DeleteResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::delete(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DeleteSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/RenameFile" => { + #[allow(non_camel_case_types)] + struct RenameFileSvc(pub Arc); + impl tonic::server::UnaryService for RenameFileSvc { + type Response = super::RenameFileResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::rename_file(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RenameFileSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/Write" => { + #[allow(non_camel_case_types)] + struct WriteSvc(pub Arc); + impl tonic::server::UnaryService for WriteSvc { + type Response = super::WriteResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::write(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = WriteSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/ReadAt" => { + #[allow(non_camel_case_types)] + struct ReadAtSvc(pub Arc); + impl tonic::server::UnaryService for ReadAtSvc { + type Response = super::ReadAtResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::read_at(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ReadAtSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/ListDir" => { + #[allow(non_camel_case_types)] + struct ListDirSvc(pub Arc); + impl tonic::server::UnaryService for ListDirSvc { + type Response = super::ListDirResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::list_dir(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ListDirSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/WalkDir" => { + #[allow(non_camel_case_types)] + struct WalkDirSvc(pub Arc); + impl tonic::server::UnaryService for WalkDirSvc { + type Response = super::WalkDirResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::walk_dir(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = WalkDirSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/RenameData" => { + #[allow(non_camel_case_types)] + struct RenameDataSvc(pub Arc); + impl tonic::server::UnaryService for RenameDataSvc { + type Response = super::RenameDataResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::rename_data(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = RenameDataSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/MakeVolumes" => { + #[allow(non_camel_case_types)] + struct MakeVolumesSvc(pub Arc); + impl tonic::server::UnaryService for MakeVolumesSvc { + type Response = super::MakeVolumesResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::make_volumes(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = MakeVolumesSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/MakeVolume" => { + #[allow(non_camel_case_types)] + struct MakeVolumeSvc(pub Arc); + impl tonic::server::UnaryService for MakeVolumeSvc { + type Response = super::MakeVolumeResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::make_volume(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = MakeVolumeSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/ListVolumes" => { + #[allow(non_camel_case_types)] + struct ListVolumesSvc(pub Arc); + impl tonic::server::UnaryService for ListVolumesSvc { + type Response = super::ListVolumesResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::list_volumes(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ListVolumesSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/StatVolume" => { + #[allow(non_camel_case_types)] + struct StatVolumeSvc(pub Arc); + impl tonic::server::UnaryService for StatVolumeSvc { + type Response = super::StatVolumeResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::stat_volume(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = StatVolumeSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/WriteMetadata" => { + #[allow(non_camel_case_types)] + struct WriteMetadataSvc(pub Arc); + impl tonic::server::UnaryService for WriteMetadataSvc { + type Response = super::WriteMetadataResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::write_metadata(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = WriteMetadataSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/ReadVersion" => { + #[allow(non_camel_case_types)] + struct ReadVersionSvc(pub Arc); + impl tonic::server::UnaryService for ReadVersionSvc { + type Response = super::ReadVersionResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::read_version(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ReadVersionSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/ReadXL" => { + #[allow(non_camel_case_types)] + struct ReadXLSvc(pub Arc); + impl tonic::server::UnaryService for ReadXLSvc { + type Response = super::ReadXlResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::read_xl(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ReadXLSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/ReadMultiple" => { + #[allow(non_camel_case_types)] + struct ReadMultipleSvc(pub Arc); + impl tonic::server::UnaryService for ReadMultipleSvc { + type Response = super::ReadMultipleResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::read_multiple(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ReadMultipleSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/DeleteVolume" => { + #[allow(non_camel_case_types)] + struct DeleteVolumeSvc(pub Arc); + impl tonic::server::UnaryService for DeleteVolumeSvc { + type Response = super::DeleteVolumeResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::delete_volume(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DeleteVolumeSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } _ => Box::pin(async move { Ok(http::Response::builder() .status(200) diff --git a/common/protos/src/node.proto b/common/protos/src/node.proto index c3d6cd29..cc3ed1be 100644 --- a/common/protos/src/node.proto +++ b/common/protos/src/node.proto @@ -26,9 +26,259 @@ message MakeBucketResponse { optional string error_info = 2; } +message ReadAllRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; + string path = 3; +} + +message ReadAllResponse { + bool success = 1; + bytes data = 2; + optional string error_info = 3; +} + +message WriteAllRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; + string path = 3; + bytes data = 4; +} + +message WriteAllResponse { + bool success = 1; + optional string error_info = 2; +} + +message DeleteRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; + string path = 3; + string options = 4; +} + +message DeleteResponse { + bool success = 1; + optional string error_info = 2; +} + +message RenameFileRequst { + string disk = 1; + string src_volume = 2; + string src_path = 3; + string dst_volume = 4; + string dst_path = 5; +} + +message RenameFileResponse { + bool success = 1; + optional string error_info = 2; +} + +message WriteRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; + string path = 3; + bool is_append = 4; + bytes data = 5; +} + +message WriteResponse { + bool success = 1; + optional string error_info = 2; +} + +// message AppendRequest { +// string disk = 1; // indicate which one in the disks +// string volume = 2; +// string path = 3; +// bytes data = 4; +// } +// +// message AppendResponse { +// bool success = 1; +// optional string error_info = 2; +// } + +message ReadAtRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; + string path = 3; + int64 offset = 4; + int64 length = 5; +} + +message ReadAtResponse { + bool success = 1; + bytes data = 2; + int64 read_size = 3; + optional string error_info = 4; +} + +message ListDirRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; +} + +message ListDirResponse { + bool success = 1; + repeated string volumes = 2; + optional string error_info = 3; +} + +message WalkDirRequest { + string disk = 1; // indicate which one in the disks + string walk_dir_options = 2; +} + +message WalkDirResponse { + bool success = 1; + repeated string meta_cache_entry = 2; + optional string error_info = 3; +} + +message RenameDataRequest { + string disk = 1; // indicate which one in the disks + string src_volume = 2; + string src_path = 3; + string file_info = 4; + string dst_volume = 5; + string dst_path = 6; +} + +message RenameDataResponse { + bool success = 1; + string rename_data_resp = 2; + optional string error_info = 3; +} + +message MakeVolumesRequest { + string disk = 1; // indicate which one in the disks + repeated string volumes = 2; +} + +message MakeVolumesResponse { + bool success = 1; + optional string error_info = 2; +} + +message MakeVolumeRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; +} + +message MakeVolumeResponse { + bool success = 1; + optional string error_info = 2; +} + +message ListVolumesRequest { + string disk = 1; // indicate which one in the disks +} + +message ListVolumesResponse { + bool success = 1; + repeated string volume_infos = 2; + optional string error_info = 3; +} + +message StatVolumeRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; +} + +message StatVolumeResponse { + bool success = 1; + string volume_info = 2; + optional string error_info = 3; +} + +message WriteMetadataRequest { + string disk = 1; // indicate which one in the disks + string volume = 2; + string path = 3; + string file_info = 4; +} + +message WriteMetadataResponse { + bool success = 1; + optional string error_info = 2; +} + +message ReadVersionRequest { + string disk = 1; + string volume = 2; + string path = 3; + string version_id = 4; + string opts = 5; +} + +message ReadVersionResponse { + bool success = 1; + string file_info = 2; + optional string error_info = 3; +} + +message ReadXLRequest { + string disk = 1; + string volume = 2; + string path = 3; + bool read_data = 4; +} + +message ReadXLResponse { + bool success = 1; + string raw_file_info = 2; + optional string error_info = 3; +} + +message ReadMultipleRequest { + string disk = 1; + string read_multiple_req = 2; +} + +message ReadMultipleResponse { + bool success = 1; + repeated string read_multiple_resps = 2; + optional string error_info = 3; +} + +message DeleteVolumeRequest { + string disk = 1; + string volume = 2; +} + +message DeleteVolumeResponse { + bool success = 1; + optional string error_info = 2; +} + /* -------------------------------------------------------------------- */ service NodeService { +/* -------------------------------meta service-------------------------- */ rpc Ping(PingRequest) returns (PingResponse) {}; rpc MakeBucket(MakeBucketRequest) returns (MakeBucketResponse) {}; -} \ No newline at end of file + +/* -------------------------------disk service-------------------------- */ + + rpc ReadAll(ReadAllRequest) returns (ReadAllResponse) {}; + rpc WriteAll(WriteAllRequest) returns (WriteAllResponse) {}; + rpc Delete(DeleteRequest) returns (DeleteResponse) {}; + rpc RenameFile(RenameFileRequst) returns (RenameFileResponse) {}; + rpc Write(WriteRequest) returns (WriteResponse) {}; +// rpc Append(AppendRequest) returns (AppendResponse) {}; + rpc ReadAt(ReadAtRequest) returns (ReadAtResponse) {}; + rpc ListDir(ListDirRequest) returns (ListDirResponse) {}; + rpc WalkDir(WalkDirRequest) returns (WalkDirResponse) {}; + rpc RenameData(RenameDataRequest) returns (RenameDataResponse) {}; + rpc MakeVolumes(MakeVolumesRequest) returns (MakeVolumesResponse) {}; + rpc MakeVolume(MakeVolumeRequest) returns (MakeVolumeResponse) {}; + rpc ListVolumes(ListVolumesRequest) returns (ListVolumesResponse) {}; + rpc StatVolume(StatVolumeRequest) returns (StatVolumeResponse) {}; + rpc WriteMetadata(WriteMetadataRequest) returns (WriteMetadataResponse) {}; + rpc ReadVersion(ReadVersionRequest) returns (ReadVersionResponse) {}; + rpc ReadXL(ReadXLRequest) returns (ReadXLResponse) {}; + rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {}; + rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {}; +} diff --git a/e2e_test/Cargo.toml b/e2e_test/Cargo.toml index a1e30a55..90f99158 100644 --- a/e2e_test/Cargo.toml +++ b/e2e_test/Cargo.toml @@ -9,7 +9,9 @@ rust-version.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] +ecstore.workspace = true flatbuffers.workspace = true protos.workspace = true +serde_json.workspace = true tonic = { version = "0.12.1", features = ["gzip"] } tokio = { workspace = true } \ No newline at end of file diff --git a/e2e_test/src/reliant/node_interact_test.rs b/e2e_test/src/reliant/node_interact_test.rs index b2f97e13..ca6930e8 100644 --- a/e2e_test/src/reliant/node_interact_test.rs +++ b/e2e_test/src/reliant/node_interact_test.rs @@ -1,14 +1,21 @@ #![cfg(test)] +use ecstore::disk::VolumeInfo; use protos::{ models::{PingBody, PingBodyBuilder}, - proto_gen::node_service::{node_service_client::NodeServiceClient, PingRequest, PingResponse}, + proto_gen::node_service::{ + node_service_client::NodeServiceClient, ListVolumesRequest, MakeVolumeRequest, PingRequest, PingResponse, + }, }; use std::error::Error; use tonic::Request; +async fn get_client() -> Result, Box> { + Ok(NodeServiceClient::connect("http://localhost:9000").await?) +} + #[tokio::test] -async fn main() -> Result<(), Box> { +async fn ping() -> Result<(), Box> { let mut fbb = flatbuffers::FlatBufferBuilder::new(); let payload = fbb.create_vector(b"hello world"); @@ -44,3 +51,38 @@ async fn main() -> Result<(), Box> { Ok(()) } + +#[tokio::test] +async fn make_volume() -> Result<(), Box> { + let mut client = get_client().await?; + let request = Request::new(MakeVolumeRequest { + disk: "data".to_string(), + volume: "dandan".to_string(), + }); + + let response = client.make_volume(request).await?.into_inner(); + if response.success { + println!("success"); + } else { + println!("failed: {:?}", response.error_info); + } + Ok(()) +} + +#[tokio::test] +async fn list_volumes() -> Result<(), Box> { + let mut client = get_client().await?; + let request = Request::new(ListVolumesRequest { + disk: "data".to_string(), + }); + + let response = client.list_volumes(request).await?.into_inner(); + let volume_infos: Vec = response + .volume_infos + .into_iter() + .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) + .collect(); + + println!("{:?}", volume_infos); + Ok(()) +} diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 9de369ac..aae55759 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -3,7 +3,7 @@ use super::{ DeleteOptions, DiskAPI, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, VolumeInfo, WalkDirOptions, }; -use crate::disk::STORAGE_FORMAT_FILE; +use crate::disk::{LocalFileReader, LocalFileWriter, STORAGE_FORMAT_FILE}; use crate::{ error::{Error, Result}, file_meta::FileMeta, @@ -344,6 +344,10 @@ impl DiskAPI for LocalDisk { self.id } + fn path(&self) -> PathBuf { + self.root.clone() + } + #[must_use] async fn read_all(&self, volume: &str, path: &str) -> Result { let p = self.get_object_path(volume, path)?; @@ -443,7 +447,8 @@ impl DiskAPI for LocalDisk { let file = File::create(&fpath).await?; - Ok(FileWriter::new(file)) + Ok(FileWriter::Local(LocalFileWriter::new(file))) + // Ok(FileWriter::new(file)) // let mut writer = BufWriter::new(file); @@ -469,7 +474,8 @@ impl DiskAPI for LocalDisk { .open(&p) .await?; - Ok(FileWriter::new(file)) + Ok(FileWriter::Local(LocalFileWriter::new(file))) + // Ok(FileWriter::new(file)) // let mut writer = BufWriter::new(file); @@ -486,7 +492,7 @@ impl DiskAPI for LocalDisk { debug!("read_file {:?}", &p); let file = File::options().read(true).open(&p).await?; - Ok(FileReader::new(file)) + Ok(FileReader::Local(LocalFileReader::new(file))) // file.seek(SeekFrom::Start(offset as u64)).await?; @@ -683,9 +689,7 @@ impl DiskAPI for LocalDisk { .await?; } - Ok(RenameDataResp { - old_data_dir: old_data_dir, - }) + Ok(RenameDataResp { old_data_dir }) } async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index b694f99b..2fc1cf3c 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -2,6 +2,7 @@ pub mod endpoint; pub mod error; pub mod format; mod local; +mod remote; pub const RUSTFS_META_BUCKET: &str = ".rustfs.sys"; pub const RUSTFS_META_MULTIPART_BUCKET: &str = ".rustfs.sys/multipart"; @@ -12,18 +13,22 @@ pub const FORMAT_CONFIG_FILE: &str = "format.json"; const STORAGE_FORMAT_FILE: &str = "xl.meta"; use crate::{ - erasure::ReadAt, + erasure::{ReadAt, Write}, error::Result, file_meta::FileMeta, store_api::{FileInfo, RawFileInfo}, }; use bytes::Bytes; -use std::{fmt::Debug, io::SeekFrom, pin::Pin, sync::Arc}; +use protos::proto_gen::node_service::{node_service_client::NodeServiceClient, ReadAtRequest, WriteRequest}; +use serde::{Deserialize, Serialize}; +use std::{fmt::Debug, io::SeekFrom, path::PathBuf, sync::Arc}; use time::OffsetDateTime; use tokio::{ fs::File, - io::{AsyncReadExt, AsyncSeekExt, AsyncWrite}, + io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, }; +use tonic::{transport::Channel, Request}; +use tower::timeout::Timeout; use uuid::Uuid; pub type DiskStore = Arc>; @@ -33,8 +38,8 @@ pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result Result bool; fn id(&self) -> Uuid; + fn path(&self) -> PathBuf; async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; async fn read_all(&self, volume: &str, path: &str) -> Result; @@ -82,7 +88,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn read_multiple(&self, req: ReadMultipleReq) -> Result>; } -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct WalkDirOptions { // Bucket to scanner pub bucket: String, @@ -109,7 +115,7 @@ pub struct WalkDirOptions { pub disk_id: String, } -#[derive(Debug, Default)] +#[derive(Debug, Default, Serialize, Deserialize)] pub struct MetaCacheEntry { // name is the full name of the object including prefixes pub name: String, @@ -184,17 +190,18 @@ pub struct DiskOption { pub health_check: bool, } +#[derive(Serialize, Deserialize)] pub struct RenameDataResp { pub old_data_dir: Option, } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct DeleteOptions { pub recursive: bool, pub immediate: bool, } -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ReadMultipleReq { pub bucket: String, pub prefix: String, @@ -205,7 +212,7 @@ pub struct ReadMultipleReq { pub max_results: usize, } -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct ReadMultipleResp { pub bucket: String, pub prefix: String, @@ -230,65 +237,160 @@ pub struct ReadMultipleResp { // } // } +#[derive(Debug, Deserialize, Serialize)] pub struct VolumeInfo { pub name: String, pub created: Option, } +#[derive(Deserialize, Serialize)] pub struct ReadOptions { pub read_data: bool, pub healing: bool, } -pub struct FileWriter { - pub inner: Pin>, +// pub struct FileWriter { +// pub inner: Pin>, +// } + +// impl AsyncWrite for FileWriter { +// fn poll_write( +// mut self: Pin<&mut Self>, +// cx: &mut std::task::Context<'_>, +// buf: &[u8], +// ) -> std::task::Poll> { +// Pin::new(&mut self.inner).poll_write(cx, buf) +// } + +// fn poll_flush( +// mut self: Pin<&mut Self>, +// cx: &mut std::task::Context<'_>, +// ) -> std::task::Poll> { +// Pin::new(&mut self.inner).poll_flush(cx) +// } + +// fn poll_shutdown( +// mut self: Pin<&mut Self>, +// cx: &mut std::task::Context<'_>, +// ) -> std::task::Poll> { +// Pin::new(&mut self.inner).poll_shutdown(cx) +// } +// } + +// impl FileWriter { +// pub fn new(inner: W) -> Self +// where +// W: AsyncWrite + Send + Sync + 'static, +// { +// Self { inner: Box::pin(inner) } +// } +// } + +pub enum FileWriter { + Local(LocalFileWriter), + Remote(RemoteFileWriter), } -impl AsyncWrite for FileWriter { - fn poll_write( - mut self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - buf: &[u8], - ) -> std::task::Poll> { - Pin::new(&mut self.inner).poll_write(cx, buf) - } - - fn poll_flush( - mut self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - Pin::new(&mut self.inner).poll_flush(cx) - } - - fn poll_shutdown( - mut self: Pin<&mut Self>, - cx: &mut std::task::Context<'_>, - ) -> std::task::Poll> { - Pin::new(&mut self.inner).poll_shutdown(cx) +#[async_trait::async_trait] +impl Write for FileWriter { + async fn write(&mut self, buf: &[u8]) -> Result<()> { + match self { + Self::Local(local_file_writer) => local_file_writer.write(buf).await, + Self::Remote(remote_file_writer) => remote_file_writer.write(buf).await, + } } } -impl FileWriter { - pub fn new(inner: W) -> Self - where - W: AsyncWrite + Send + Sync + 'static, - { - Self { inner: Box::pin(inner) } - } -} - -#[derive(Debug)] -pub struct FileReader { +pub struct LocalFileWriter { pub inner: File, } -impl FileReader { +impl LocalFileWriter { pub fn new(inner: File) -> Self { Self { inner } } } +#[async_trait::async_trait] +impl Write for LocalFileWriter { + async fn write(&mut self, buf: &[u8]) -> Result<()> { + self.inner.write(buf).await?; + self.inner.flush().await?; + + Ok(()) + } +} + +pub struct RemoteFileWriter { + pub root: PathBuf, + pub volume: String, + pub path: String, + pub is_append: bool, + client: NodeServiceClient>, +} + +impl RemoteFileWriter { + pub fn new( + root: PathBuf, + volume: String, + path: String, + is_append: bool, + client: NodeServiceClient>, + ) -> Self { + Self { + root, + volume, + path, + is_append, + client, + } + } +} + +#[async_trait::async_trait] +impl Write for RemoteFileWriter { + async fn write(&mut self, buf: &[u8]) -> Result<()> { + let request = Request::new(WriteRequest { + disk: self.root.to_string_lossy().to_string(), + volume: self.volume.to_string(), + path: self.path.to_string(), + is_append: self.is_append, + data: buf.to_vec(), + }); + let _response = self.client.write(request).await?.into_inner(); + Ok(()) + } +} + +#[derive(Debug)] +pub enum FileReader { + Local(LocalFileReader), + Remote(RemoteFileReader), +} + +#[async_trait::async_trait] impl ReadAt for FileReader { + async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { + match self { + Self::Local(local_file_writer) => local_file_writer.read_at(offset, length).await, + Self::Remote(remote_file_writer) => remote_file_writer.read_at(offset, length).await, + } + } +} + +#[derive(Debug)] +pub struct LocalFileReader { + pub inner: File, +} + +impl LocalFileReader { + pub fn new(inner: File) -> Self { + Self { inner } + } +} + +#[async_trait::async_trait] +impl ReadAt for LocalFileReader { async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { self.inner.seek(SeekFrom::Start(offset as u64)).await?; @@ -301,3 +403,38 @@ impl ReadAt for FileReader { Ok((buffer, bytes_read)) } } + +#[derive(Debug)] +pub struct RemoteFileReader { + pub root: PathBuf, + pub volume: String, + pub path: String, + client: NodeServiceClient>, +} + +impl RemoteFileReader { + pub fn new(root: PathBuf, volume: String, path: String, client: NodeServiceClient>) -> Self { + Self { + root, + volume, + path, + client, + } + } +} + +#[async_trait::async_trait] +impl ReadAt for RemoteFileReader { + async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)> { + let request = Request::new(ReadAtRequest { + disk: self.root.to_string_lossy().to_string(), + volume: self.volume.to_string(), + path: self.path.to_string(), + offset: offset.try_into().unwrap(), + length: length.try_into().unwrap(), + }); + let response = self.client.read_at(request).await?.into_inner(); + + Ok((response.data, response.read_size.try_into().unwrap())) + } +} diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs new file mode 100644 index 00000000..d1803d21 --- /dev/null +++ b/ecstore/src/disk/remote.rs @@ -0,0 +1,356 @@ +use std::{path::PathBuf, time::Duration}; + +use bytes::Bytes; +use protos::{ + node_service_time_out_client, + proto_gen::node_service::{ + node_service_client::NodeServiceClient, DeleteRequest, DeleteVolumeRequest, ListDirRequest, ListVolumesRequest, + MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, + RenameDataRequest, RenameFileRequst, StatVolumeRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest, + }, + DEFAULT_GRPC_SERVER_MESSAGE_LEN, +}; +use tokio::fs; +use tonic::{ + transport::{Channel, Endpoint as tonic_Endpoint}, + Request, +}; +use tower::timeout::Timeout; +use uuid::Uuid; + +use crate::{ + error::Result, + store_api::{FileInfo, RawFileInfo}, +}; + +use super::{ + endpoint::Endpoint, DeleteOptions, DiskAPI, DiskOption, FileReader, FileWriter, MetaCacheEntry, ReadMultipleReq, + ReadMultipleResp, ReadOptions, RemoteFileReader, RemoteFileWriter, RenameDataResp, VolumeInfo, WalkDirOptions, +}; + +#[derive(Debug)] +pub struct RemoteDisk { + channel: Channel, + pub root: PathBuf, +} + +impl RemoteDisk { + pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result { + let connector = tonic_Endpoint::from_shared(format!( + "{}://{}{}", + ep.url.scheme(), + ep.url.host_str().unwrap(), + ep.url.port().unwrap() + )) + .unwrap(); + let channel = tokio::runtime::Runtime::new().unwrap().block_on(connector.connect()).unwrap(); + + let root = fs::canonicalize(ep.url.path()).await?; + + Ok(Self { channel, root }) + } + + fn get_client(&self) -> NodeServiceClient> { + node_service_time_out_client( + self.channel.clone(), + Duration::new(30, 0), // TODO: use config setting + DEFAULT_GRPC_SERVER_MESSAGE_LEN, + // grpc_enable_gzip, + false, // TODO: use config setting + ) + } +} + +// TODO: all api need to handle errors +#[async_trait::async_trait] +impl DiskAPI for RemoteDisk { + fn is_local(&self) -> bool { + false + } + + fn id(&self) -> Uuid { + Uuid::nil() + } + + fn path(&self) -> PathBuf { + self.root.clone() + } + + async fn read_all(&self, volume: &str, path: &str) -> Result { + let mut client = self.get_client(); + let request = Request::new(ReadAllRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + path: path.to_string(), + }); + + let response = client.read_all(request).await?.into_inner(); + + Ok(Bytes::from(response.data)) + } + + async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { + let mut client = self.get_client(); + let request = Request::new(WriteAllRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + path: path.to_string(), + data, + }); + + let _response = client.write_all(request).await?.into_inner(); + + Ok(()) + } + + async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { + let options = serde_json::to_string(&opt)?; + let mut client = self.get_client(); + let request = Request::new(DeleteRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + path: path.to_string(), + options, + }); + + let _response = client.delete(request).await?.into_inner(); + + Ok(()) + } + + async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { + let mut client = self.get_client(); + let request = Request::new(RenameFileRequst { + disk: self.root.to_string_lossy().to_string(), + src_volume: src_volume.to_string(), + src_path: src_path.to_string(), + dst_volume: dst_volume.to_string(), + dst_path: dst_path.to_string(), + }); + + let _response = client.rename_file(request).await?.into_inner(); + + Ok(()) + } + + async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result { + Ok(FileWriter::Remote(RemoteFileWriter::new( + self.root.clone(), + volume.to_string(), + path.to_string(), + false, + self.get_client(), + ))) + } + + async fn append_file(&self, volume: &str, path: &str) -> Result { + Ok(FileWriter::Remote(RemoteFileWriter::new( + self.root.clone(), + volume.to_string(), + path.to_string(), + true, + self.get_client(), + ))) + } + + async fn read_file(&self, volume: &str, path: &str) -> Result { + Ok(FileReader::Remote(RemoteFileReader::new( + self.root.clone(), + volume.to_string(), + path.to_string(), + self.get_client(), + ))) + } + + async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result> { + let mut client = self.get_client(); + let request = Request::new(ListDirRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + }); + + let response = client.list_dir(request).await?.into_inner(); + + Ok(response.volumes) + } + + async fn walk_dir(&self, opts: WalkDirOptions) -> Result> { + let walk_dir_options = serde_json::to_string(&opts)?; + let mut client = self.get_client(); + let request = Request::new(WalkDirRequest { + disk: self.root.to_string_lossy().to_string(), + walk_dir_options, + }); + + let response = client.walk_dir(request).await?.into_inner(); + let entries = response + .meta_cache_entry + .into_iter() + .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) + .collect(); + + Ok(entries) + } + + async fn rename_data( + &self, + src_volume: &str, + src_path: &str, + fi: FileInfo, + dst_volume: &str, + dst_path: &str, + ) -> Result { + let file_info = serde_json::to_string(&fi)?; + let mut client = self.get_client(); + let request = Request::new(RenameDataRequest { + disk: self.root.to_string_lossy().to_string(), + src_volume: src_volume.to_string(), + src_path: src_path.to_string(), + file_info, + dst_volume: dst_volume.to_string(), + dst_path: dst_path.to_string(), + }); + + let response = client.rename_data(request).await?.into_inner(); + let rename_data_resp = serde_json::from_str::(&response.rename_data_resp)?; + + Ok(rename_data_resp) + } + + async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { + let mut client = self.get_client(); + let request = Request::new(MakeVolumesRequest { + disk: self.root.to_string_lossy().to_string(), + volumes: volumes.iter().map(|s| (*s).to_string()).collect(), + }); + + let _response = client.make_volumes(request).await?.into_inner(); + + Ok(()) + } + + async fn make_volume(&self, volume: &str) -> Result<()> { + let mut client = self.get_client(); + let request = Request::new(MakeVolumeRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + }); + + let _response = client.make_volume(request).await?.into_inner(); + + Ok(()) + } + + async fn list_volumes(&self) -> Result> { + let mut client = self.get_client(); + let request = Request::new(ListVolumesRequest { + disk: self.root.to_string_lossy().to_string(), + }); + + let response = client.list_volumes(request).await?.into_inner(); + let infos = response + .volume_infos + .into_iter() + .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) + .collect(); + + Ok(infos) + } + + async fn stat_volume(&self, volume: &str) -> Result { + let mut client = self.get_client(); + let request = Request::new(StatVolumeRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + }); + + let response = client.stat_volume(request).await?.into_inner(); + let volume_info = serde_json::from_str::(&response.volume_info)?; + + Ok(volume_info) + } + + async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { + let file_info = serde_json::to_string(&fi)?; + let mut client = self.get_client(); + let request = Request::new(WriteMetadataRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + path: path.to_string(), + file_info, + }); + + let _response = client.write_metadata(request).await?.into_inner(); + + Ok(()) + } + + async fn read_version( + &self, + _org_volume: &str, + volume: &str, + path: &str, + version_id: &str, + opts: &ReadOptions, + ) -> Result { + let opts = serde_json::to_string(opts)?; + let mut client = self.get_client(); + let request = Request::new(ReadVersionRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + path: path.to_string(), + version_id: version_id.to_string(), + opts, + }); + + let response = client.read_version(request).await?.into_inner(); + let file_info = serde_json::from_str::(&response.file_info)?; + + Ok(file_info) + } + + async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { + let mut client = self.get_client(); + let request = Request::new(ReadXlRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + path: path.to_string(), + read_data, + }); + + let response = client.read_xl(request).await?.into_inner(); + let raw_file_info = serde_json::from_str::(&response.raw_file_info)?; + + Ok(raw_file_info) + } + + async fn read_multiple(&self, req: ReadMultipleReq) -> Result> { + let read_multiple_req = serde_json::to_string(&req)?; + let mut client = self.get_client(); + let request = Request::new(ReadMultipleRequest { + disk: self.root.to_string_lossy().to_string(), + read_multiple_req, + }); + + let response = client.read_multiple(request).await?.into_inner(); + let read_multiple_resps = response + .read_multiple_resps + .into_iter() + .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) + .collect(); + + Ok(read_multiple_resps) + } + + async fn delete_volume(&self, volume: &str) -> Result<()> { + let mut client = self.get_client(); + let request = Request::new(DeleteVolumeRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + }); + + let _response = client.delete_volume(request).await?.into_inner(); + + Ok(()) + } +} diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 6c23842a..13cb5af1 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -3,7 +3,7 @@ use bytes::Bytes; use futures::future::join_all; use futures::{Stream, StreamExt}; use reed_solomon_erasure::galois_8::ReedSolomon; -use tokio::io::AsyncWrite; +use std::fmt::Debug; use tokio::io::AsyncWriteExt; use tokio::io::DuplexStream; use tracing::debug; @@ -13,7 +13,7 @@ use uuid::Uuid; use crate::chunk_stream::ChunkedStream; use crate::disk::error::DiskError; -use crate::disk::FileReader; +use crate::disk::{FileReader, FileWriter}; pub struct Erasure { data_shards: usize, @@ -43,17 +43,16 @@ impl Erasure { } } - pub async fn encode( + pub async fn encode( &self, body: S, - writers: &mut [W], + writers: &mut [FileWriter], // block_size: usize, total_size: usize, _write_quorum: usize, ) -> Result where S: Stream> + Send + Sync + 'static, - W: AsyncWrite + Unpin, { let mut stream = ChunkedStream::new(body, total_size, self.block_size, false); let mut total: usize = 0; @@ -85,7 +84,7 @@ impl Erasure { let mut errs = Vec::new(); for (i, w) in writers.iter_mut().enumerate() { - match w.write_all(blocks[i].as_ref()).await { + match w.write(blocks[i].as_ref()).await { Ok(_) => errs.push(None), Err(e) => errs.push(Some(e)), } @@ -318,6 +317,12 @@ impl Erasure { } } +#[async_trait::async_trait] +pub trait Write { + async fn write(&mut self, buf: &[u8]) -> Result<()>; +} + +#[async_trait::async_trait] pub trait ReadAt { async fn read_at(&mut self, offset: usize, length: usize) -> Result<(Vec, usize)>; } diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 82741b44..beec2d9e 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -3,7 +3,7 @@ mod chunk_stream; pub mod disk; mod disks_layout; mod endpoints; -mod erasure; +pub mod erasure; pub mod error; mod file_meta; pub mod peer; diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index a1556b10..f0f0e673 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -197,6 +197,7 @@ pub struct ObjectPartInfo { // } // } +#[derive(Serialize, Deserialize)] pub struct RawFileInfo { pub buf: Vec, } diff --git a/rustfs/Cargo.toml b/rustfs/Cargo.toml index a155cc22..89bd92a7 100644 --- a/rustfs/Cargo.toml +++ b/rustfs/Cargo.toml @@ -28,6 +28,7 @@ prost-types.workspace = true protos.workspace = true protobuf.workspace = true s3s.workspace = true +serde_json.workspace = true tracing.workspace = true time = { workspace = true, features = ["parsing", "formatting"] } tokio = { workspace = true, features = [ diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 890e6cba..7fa7d5de 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -1,4 +1,10 @@ -use ecstore::{disk::DiskStore, peer::LocalPeerS3Client}; +use ecstore::{ + disk::{DeleteOptions, DiskStore, ReadMultipleReq, ReadOptions, WalkDirOptions}, + erasure::{ReadAt, Write}, + peer::{LocalPeerS3Client, PeerS3Client}, + store_api::{FileInfo, MakeBucketOptions}, +}; +use tokio::fs; use tonic::{Request, Response, Status}; use tracing::{debug, error, info}; @@ -6,7 +12,13 @@ use protos::{ models::{PingBody, PingBodyBuilder}, proto_gen::node_service::{ node_service_server::{NodeService as Node, NodeServiceServer as NodeServer}, - MakeBucketRequest, MakeBucketResponse, PingRequest, PingResponse, + DeleteRequest, DeleteResponse, DeleteVolumeRequest, DeleteVolumeResponse, ListDirRequest, ListDirResponse, + ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, + MakeVolumesRequest, MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, + ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, + ReadXlResponse, RenameDataRequest, RenameDataResponse, RenameFileRequst, RenameFileResponse, StatVolumeRequest, + StatVolumeResponse, WalkDirRequest, WalkDirResponse, WriteAllRequest, WriteAllResponse, WriteMetadataRequest, + WriteMetadataResponse, WriteRequest, WriteResponse, }, }; @@ -20,6 +32,24 @@ pub fn make_server(local_disks: Vec) -> NodeServer { NodeServer::new(NodeService { local_peer }) } +impl NodeService { + async fn find_disk(&self, disk_path: &String) -> Option { + let disk_path = match fs::canonicalize(disk_path).await { + Ok(disk_path) => disk_path, + Err(_) => return None, + }; + self.local_peer.local_disks.iter().find(|&x| x.path() == disk_path).cloned() + } + + fn all_disk(&self) -> Vec { + self.local_peer + .local_disks + .iter() + .map(|disk| disk.path().to_string_lossy().to_string()) + .collect() + } +} + #[tonic::async_trait] impl Node for NodeService { async fn ping(&self, request: Request) -> Result, Status> { @@ -52,9 +82,578 @@ impl Node for NodeService { async fn make_bucket(&self, request: Request) -> Result, Status> { debug!("make bucket"); - let _req = request.into_inner(); + let make_bucket_request = request.into_inner(); + let options = if let Some(opts) = make_bucket_request.options { + MakeBucketOptions { + force_create: opts.force_create, + } + } else { + MakeBucketOptions::default() + }; + match self.local_peer.make_bucket(&make_bucket_request.name, &options).await { + Ok(_) => Ok(tonic::Response::new(MakeBucketResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(MakeBucketResponse { + success: false, + error_info: Some(format!("make failed: {}", err.to_string())), + })), + } + } - // match self.local_peer.make_bucket(&name, &MakeBucketOptions::default()).await {} - unimplemented!() + async fn read_all(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk.read_all(&request.volume, &request.path).await { + Ok(data) => Ok(tonic::Response::new(ReadAllResponse { + success: true, + data: data.to_vec(), + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(ReadAllResponse { + success: false, + data: Vec::new(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(ReadAllResponse { + success: false, + data: Vec::new(), + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn write_all(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk.write_all(&request.volume, &request.path, request.data).await { + Ok(_) => Ok(tonic::Response::new(WriteAllResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(WriteAllResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(WriteAllResponse { + success: false, + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn delete(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let options = match serde_json::from_str::(&request.options) { + Ok(options) => options, + Err(_) => { + return Ok(tonic::Response::new(DeleteResponse { + success: false, + error_info: Some("can not decode DeleteOptions".to_string()), + })); + } + }; + match disk.delete(&request.volume, &request.path, options).await { + Ok(_) => Ok(tonic::Response::new(DeleteResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(DeleteResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(DeleteResponse { + success: false, + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn rename_file(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk + .rename_file(&request.src_volume, &request.src_path, &request.dst_volume, &request.dst_path) + .await + { + Ok(_) => Ok(tonic::Response::new(RenameFileResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(RenameFileResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(RenameFileResponse { + success: false, + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn write(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let file_writer = if request.is_append { + disk.append_file(&request.volume, &request.path).await + } else { + disk.create_file("", &request.volume, &request.path, 0).await + }; + + match file_writer { + Ok(mut file_writer) => match file_writer.write(&request.data).await { + Ok(_) => Ok(tonic::Response::new(WriteResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(WriteResponse { + success: false, + error_info: Some(err.to_string()), + })), + }, + Err(err) => Ok(tonic::Response::new(WriteResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(WriteResponse { + success: false, + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn read_at(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk.read_file(&request.volume, &request.path).await { + Ok(mut file_reader) => { + match file_reader + .read_at(request.offset.try_into().unwrap(), request.length.try_into().unwrap()) + .await + { + Ok((data, read_size)) => Ok(tonic::Response::new(ReadAtResponse { + success: true, + data, + read_size: read_size.try_into().unwrap(), + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(ReadAtResponse { + success: false, + data: Vec::new(), + read_size: -1, + error_info: Some(err.to_string()), + })), + } + } + Err(err) => Ok(tonic::Response::new(ReadAtResponse { + success: false, + data: Vec::new(), + read_size: -1, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(ReadAtResponse { + success: false, + data: Vec::new(), + read_size: -1, + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn list_dir(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk.list_dir("", &request.volume, "", 0).await { + Ok(volumes) => Ok(tonic::Response::new(ListDirResponse { + success: true, + volumes, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(ListDirResponse { + success: false, + volumes: Vec::new(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(ListDirResponse { + success: false, + volumes: Vec::new(), + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn walk_dir(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let opts = match serde_json::from_str::(&request.walk_dir_options) { + Ok(options) => options, + Err(_) => { + return Ok(tonic::Response::new(WalkDirResponse { + success: false, + meta_cache_entry: Vec::new(), + error_info: Some("can not decode DeleteOptions".to_string()), + })); + } + }; + match disk.walk_dir(opts).await { + Ok(entries) => { + let entries = entries + .into_iter() + .filter_map(|entry| serde_json::to_string(&entry).ok()) + .collect(); + Ok(tonic::Response::new(WalkDirResponse { + success: true, + meta_cache_entry: entries, + error_info: None, + })) + } + Err(err) => Ok(tonic::Response::new(WalkDirResponse { + success: false, + meta_cache_entry: Vec::new(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(WalkDirResponse { + success: false, + meta_cache_entry: Vec::new(), + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn rename_data(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let file_info = match serde_json::from_str::(&request.file_info) { + Ok(file_info) => file_info, + Err(_) => { + return Ok(tonic::Response::new(RenameDataResponse { + success: false, + rename_data_resp: String::new(), + error_info: Some("can not decode DeleteOptions".to_string()), + })); + } + }; + match disk + .rename_data(&request.src_volume, &request.src_path, file_info, &request.dst_volume, &request.dst_path) + .await + { + Ok(rename_data_resp) => { + let rename_data_resp = match serde_json::to_string(&rename_data_resp) { + Ok(file_info) => file_info, + Err(_) => { + return Ok(tonic::Response::new(RenameDataResponse { + success: false, + rename_data_resp: String::new(), + error_info: Some("can not encode RenameDataResp".to_string()), + })); + } + }; + Ok(tonic::Response::new(RenameDataResponse { + success: true, + rename_data_resp, + error_info: None, + })) + } + Err(err) => Ok(tonic::Response::new(RenameDataResponse { + success: false, + rename_data_resp: String::new(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(RenameDataResponse { + success: false, + rename_data_resp: String::new(), + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn make_volumes(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk.make_volumes(request.volumes.iter().map(|s| &**s).collect()).await { + Ok(_) => Ok(tonic::Response::new(MakeVolumesResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(MakeVolumesResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(MakeVolumesResponse { + success: false, + error_info: Some(format!("can not find disk, all disks: {:?}", self.all_disk())), + })) + } + } + + async fn make_volume(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk.make_volume(&request.volume).await { + Ok(_) => Ok(tonic::Response::new(MakeVolumeResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(MakeVolumeResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(MakeVolumeResponse { + success: false, + error_info: Some(format!("can not find disk, all disks: {:?}", self.all_disk())), + })) + } + } + + async fn list_volumes(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk.list_volumes().await { + Ok(volume_infos) => { + let volume_infos = volume_infos + .into_iter() + .filter_map(|volume_info| serde_json::to_string(&volume_info).ok()) + .collect(); + Ok(tonic::Response::new(ListVolumesResponse { + success: true, + volume_infos, + error_info: None, + })) + } + Err(err) => Ok(tonic::Response::new(ListVolumesResponse { + success: false, + volume_infos: Vec::new(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(ListVolumesResponse { + success: false, + volume_infos: Vec::new(), + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn stat_volume(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk.stat_volume(&request.volume).await { + Ok(volume_info) => match serde_json::to_string(&volume_info) { + Ok(volume_info) => Ok(tonic::Response::new(StatVolumeResponse { + success: true, + volume_info, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(StatVolumeResponse { + success: false, + volume_info: String::new(), + error_info: Some(format!("encode VolumeInfo failed, {}", err.to_string())), + })), + }, + Err(err) => Ok(tonic::Response::new(StatVolumeResponse { + success: false, + volume_info: String::new(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(StatVolumeResponse { + success: false, + volume_info: String::new(), + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn write_metadata(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let file_info = match serde_json::from_str::(&request.file_info) { + Ok(file_info) => file_info, + Err(err) => { + return Ok(tonic::Response::new(WriteMetadataResponse { + success: false, + error_info: Some(format!("decode FileInfo failed, {}", err.to_string())), + })); + } + }; + match disk.write_metadata("", &request.volume, &request.path, file_info).await { + Ok(_) => Ok(tonic::Response::new(WriteMetadataResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(WriteMetadataResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(WriteMetadataResponse { + success: false, + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn read_version(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let opts = match serde_json::from_str::(&request.opts) { + Ok(options) => options, + Err(_) => { + return Ok(tonic::Response::new(ReadVersionResponse { + success: false, + file_info: String::new(), + error_info: Some("can not decode DeleteOptions".to_string()), + })); + } + }; + match disk + .read_version("", &request.volume, &request.path, &request.version_id, &opts) + .await + { + Ok(file_info) => match serde_json::to_string(&file_info) { + Ok(file_info) => Ok(tonic::Response::new(ReadVersionResponse { + success: true, + file_info, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(ReadVersionResponse { + success: false, + file_info: String::new(), + error_info: Some(format!("encode VolumeInfo failed, {}", err.to_string())), + })), + }, + Err(err) => Ok(tonic::Response::new(ReadVersionResponse { + success: false, + file_info: String::new(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(ReadVersionResponse { + success: false, + file_info: String::new(), + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn read_xl(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk.read_xl(&request.volume, &request.path, request.read_data).await { + Ok(raw_file_info) => match serde_json::to_string(&raw_file_info) { + Ok(raw_file_info) => Ok(tonic::Response::new(ReadXlResponse { + success: true, + raw_file_info, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(ReadXlResponse { + success: false, + raw_file_info: String::new(), + error_info: Some(format!("encode RawFileInfo failed, {}", err.to_string())), + })), + }, + Err(err) => Ok(tonic::Response::new(ReadXlResponse { + success: false, + raw_file_info: String::new(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(ReadXlResponse { + success: false, + raw_file_info: String::new(), + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn read_multiple(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let read_multiple_req = match serde_json::from_str::(&request.read_multiple_req) { + Ok(read_multiple_req) => read_multiple_req, + Err(_) => { + return Ok(tonic::Response::new(ReadMultipleResponse { + success: false, + read_multiple_resps: Vec::new(), + error_info: Some("can not decode ReadMultipleReq".to_string()), + })); + } + }; + match disk.read_multiple(read_multiple_req).await { + Ok(read_multiple_resps) => { + let read_multiple_resps = read_multiple_resps + .into_iter() + .filter_map(|read_multiple_resp| serde_json::to_string(&read_multiple_resp).ok()) + .collect(); + + Ok(tonic::Response::new(ReadMultipleResponse { + success: true, + read_multiple_resps, + error_info: None, + })) + } + Err(err) => Ok(tonic::Response::new(ReadMultipleResponse { + success: false, + read_multiple_resps: Vec::new(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(ReadMultipleResponse { + success: false, + read_multiple_resps: Vec::new(), + error_info: Some("can not find disk".to_string()), + })) + } + } + + async fn delete_volume(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + match disk.delete_volume(&request.volume).await { + Ok(_) => Ok(tonic::Response::new(DeleteVolumeResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(DeleteVolumeResponse { + success: false, + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(DeleteVolumeResponse { + success: false, + error_info: Some("can not find disk".to_string()), + })) + } } } From 29f4eaf94f7201be27b934ab2a4b2275b2292653 Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Thu, 29 Aug 2024 19:29:13 +0800 Subject: [PATCH 07/23] support remote peer sys Signed-off-by: junxiang Mu <1948535941@qq.com> --- .../src/generated/proto_gen/node_service.rs | 193 +++++++++++++++++- common/protos/src/node.proto | 35 +++- ecstore/src/peer.rs | 67 ++++-- ecstore/src/store_api.rs | 5 +- rustfs/src/grpc.rs | 113 +++++++++- 5 files changed, 380 insertions(+), 33 deletions(-) diff --git a/common/protos/src/generated/proto_gen/node_service.rs b/common/protos/src/generated/proto_gen/node_service.rs index 13e968bf..f72dbb10 100644 --- a/common/protos/src/generated/proto_gen/node_service.rs +++ b/common/protos/src/generated/proto_gen/node_service.rs @@ -17,18 +17,28 @@ pub struct PingResponse { pub body: ::prost::alloc::vec::Vec, } #[allow(clippy::derive_partial_eq_without_eq)] -#[derive(Clone, Copy, PartialEq, ::prost::Message)] -pub struct MakeBucketOptions { +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListBucketRequest { + #[prost(string, tag = "1")] + pub options: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct ListBucketResponse { #[prost(bool, tag = "1")] - pub force_create: bool, + pub success: bool, + #[prost(string, repeated, tag = "2")] + pub bucket_infos: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] pub struct MakeBucketRequest { #[prost(string, tag = "1")] pub name: ::prost::alloc::string::String, - #[prost(message, optional, tag = "2")] - pub options: ::core::option::Option, + #[prost(string, tag = "2")] + pub options: ::prost::alloc::string::String, } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] @@ -40,6 +50,38 @@ pub struct MakeBucketResponse { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetBucketInfoRequest { + #[prost(string, tag = "1")] + pub bucket: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub options: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct GetBucketInfoResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, tag = "2")] + pub bucket_info: ::prost::alloc::string::String, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteBucketRequest { + #[prost(string, tag = "1")] + pub bucket: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteBucketResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, optional, tag = "2")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ReadAllRequest { /// indicate which one in the disks #[prost(string, tag = "1")] @@ -503,6 +545,21 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "Ping")); self.inner.unary(req, path, codec).await } + pub async fn list_bucket( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListBucket"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "ListBucket")); + self.inner.unary(req, path, codec).await + } pub async fn make_bucket( &mut self, request: impl tonic::IntoRequest, @@ -518,6 +575,36 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "MakeBucket")); self.inner.unary(req, path, codec).await } + pub async fn get_bucket_info( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/GetBucketInfo"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "GetBucketInfo")); + self.inner.unary(req, path, codec).await + } + pub async fn delete_bucket( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result, tonic::Status> { + self.inner + .ready() + .await + .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteBucket"); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "DeleteBucket")); + self.inner.unary(req, path, codec).await + } pub async fn read_all( &mut self, request: impl tonic::IntoRequest, @@ -803,10 +890,22 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn list_bucket( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn make_bucket( &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn get_bucket_info( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; + async fn delete_bucket( + &self, + request: tonic::Request, + ) -> std::result::Result, tonic::Status>; async fn read_all( &self, request: tonic::Request, @@ -979,6 +1078,34 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/ListBucket" => { + #[allow(non_camel_case_types)] + struct ListBucketSvc(pub Arc); + impl tonic::server::UnaryService for ListBucketSvc { + type Response = super::ListBucketResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::list_bucket(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = ListBucketSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/MakeBucket" => { #[allow(non_camel_case_types)] struct MakeBucketSvc(pub Arc); @@ -1007,6 +1134,62 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/GetBucketInfo" => { + #[allow(non_camel_case_types)] + struct GetBucketInfoSvc(pub Arc); + impl tonic::server::UnaryService for GetBucketInfoSvc { + type Response = super::GetBucketInfoResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::get_bucket_info(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = GetBucketInfoSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } + "/node_service.NodeService/DeleteBucket" => { + #[allow(non_camel_case_types)] + struct DeleteBucketSvc(pub Arc); + impl tonic::server::UnaryService for DeleteBucketSvc { + type Response = super::DeleteBucketResponse; + type Future = BoxFuture, tonic::Status>; + fn call(&mut self, request: tonic::Request) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { ::delete_bucket(&inner, request).await }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DeleteBucketSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config(accept_compression_encodings, send_compression_encodings) + .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/ReadAll" => { #[allow(non_camel_case_types)] struct ReadAllSvc(pub Arc); diff --git a/common/protos/src/node.proto b/common/protos/src/node.proto index cc3ed1be..1d7fe716 100644 --- a/common/protos/src/node.proto +++ b/common/protos/src/node.proto @@ -12,13 +12,19 @@ message PingResponse { bytes body = 2; } -message MakeBucketOptions { - bool force_create = 1; +message ListBucketRequest { + string options = 1; +} + +message ListBucketResponse { + bool success = 1; + repeated string bucket_infos = 2; + optional string error_info = 3; } message MakeBucketRequest { string name = 1; - MakeBucketOptions options = 2; + string options = 2; } message MakeBucketResponse { @@ -26,6 +32,26 @@ message MakeBucketResponse { optional string error_info = 2; } +message GetBucketInfoRequest { + string bucket = 1; + string options = 2; +} + +message GetBucketInfoResponse { + bool success = 1; + string bucket_info = 2; + optional string error_info = 3; +} + +message DeleteBucketRequest { + string bucket = 1; +} + +message DeleteBucketResponse { + bool success = 1; + optional string error_info = 2; +} + message ReadAllRequest { string disk = 1; // indicate which one in the disks string volume = 2; @@ -258,7 +284,10 @@ message DeleteVolumeResponse { service NodeService { /* -------------------------------meta service-------------------------- */ rpc Ping(PingRequest) returns (PingResponse) {}; + rpc ListBucket(ListBucketRequest) returns (ListBucketResponse) {}; rpc MakeBucket(MakeBucketRequest) returns (MakeBucketResponse) {}; + rpc GetBucketInfo(GetBucketInfoRequest) returns (GetBucketInfoResponse) {}; + rpc DeleteBucket(DeleteBucketRequest) returns (DeleteBucketResponse) {}; /* -------------------------------disk service-------------------------- */ diff --git a/ecstore/src/peer.rs b/ecstore/src/peer.rs index 4274c297..8d80c6dc 100644 --- a/ecstore/src/peer.rs +++ b/ecstore/src/peer.rs @@ -1,10 +1,7 @@ use async_trait::async_trait; use futures::future::join_all; -use protos::proto_gen::node_service::MakeBucketRequest; -use protos::{ - node_service_time_out_client, proto_gen::node_service::MakeBucketOptions as proto_MakeBucketOptions, - DEFAULT_GRPC_SERVER_MESSAGE_LEN, -}; +use protos::proto_gen::node_service::{DeleteBucketRequest, GetBucketInfoRequest, ListBucketRequest, MakeBucketRequest}; +use protos::{node_service_time_out_client, DEFAULT_GRPC_SERVER_MESSAGE_LEN}; use regex::Regex; use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration}; use tonic::transport::{Channel, Endpoint}; @@ -407,10 +404,27 @@ impl PeerS3Client for RemotePeerS3Client { fn get_pools(&self) -> Option> { self.pools.clone() } - async fn list_bucket(&self, _opts: &BucketOptions) -> Result> { - unimplemented!() + async fn list_bucket(&self, opts: &BucketOptions) -> Result> { + let options = serde_json::to_string(opts)?; + let mut client = node_service_time_out_client( + self.channel.clone(), + Duration::new(30, 0), // TODO: use config setting + DEFAULT_GRPC_SERVER_MESSAGE_LEN, + // grpc_enable_gzip, + false, // TODO: use config setting + ); + let request = Request::new(ListBucketRequest { options }); + let response = client.list_bucket(request).await?.into_inner(); + let bucket_infos = response + .bucket_infos + .into_iter() + .filter_map(|json_str| serde_json::from_str::(&json_str).ok()) + .collect(); + + Ok(bucket_infos) } async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { + let options = serde_json::to_string(opts)?; let mut client = node_service_time_out_client( self.channel.clone(), Duration::new(30, 0), // TODO: use config setting @@ -420,9 +434,7 @@ impl PeerS3Client for RemotePeerS3Client { ); let request = Request::new(MakeBucketRequest { name: bucket.to_string(), - options: Some(proto_MakeBucketOptions { - force_create: opts.force_create, - }), + options, }); let _response = client.make_bucket(request).await?.into_inner(); @@ -430,12 +442,39 @@ impl PeerS3Client for RemotePeerS3Client { Ok(()) } - async fn get_bucket_info(&self, _bucket: &str, _opts: &BucketOptions) -> Result { - unimplemented!() + async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result { + let options = serde_json::to_string(opts)?; + let mut client = node_service_time_out_client( + self.channel.clone(), + Duration::new(30, 0), // TODO: use config setting + DEFAULT_GRPC_SERVER_MESSAGE_LEN, + // grpc_enable_gzip, + false, // TODO: use config setting + ); + let request = Request::new(GetBucketInfoRequest { + bucket: bucket.to_string(), + options, + }); + let response = client.get_bucket_info(request).await?.into_inner(); + let bucket_info = serde_json::from_str::(&response.bucket_info)?; + + Ok(bucket_info) } - async fn delete_bucket(&self, _bucket: &str) -> Result<()> { - unimplemented!() + async fn delete_bucket(&self, bucket: &str) -> Result<()> { + let mut client = node_service_time_out_client( + self.channel.clone(), + Duration::new(30, 0), // TODO: use config setting + DEFAULT_GRPC_SERVER_MESSAGE_LEN, + // grpc_enable_gzip, + false, // TODO: use config setting + ); + let request = Request::new(DeleteBucketRequest { + bucket: bucket.to_string(), + }); + let _response = client.delete_bucket(request).await?.into_inner(); + + Ok(()) } } diff --git a/ecstore/src/store_api.rs b/ecstore/src/store_api.rs index f0f0e673..09993833 100644 --- a/ecstore/src/store_api.rs +++ b/ecstore/src/store_api.rs @@ -243,7 +243,7 @@ pub enum BitrotAlgorithm { BLAKE2b512, } -#[derive(Debug, Default)] +#[derive(Debug, Default, Serialize, Deserialize)] pub struct MakeBucketOptions { pub force_create: bool, } @@ -378,9 +378,10 @@ pub struct ObjectOptions { // } // } +#[derive(Debug, Default, Serialize, Deserialize)] pub struct BucketOptions {} -#[derive(Debug, Clone)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct BucketInfo { pub name: String, pub created: Option, diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index 7fa7d5de..544f5810 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -2,7 +2,7 @@ use ecstore::{ disk::{DeleteOptions, DiskStore, ReadMultipleReq, ReadOptions, WalkDirOptions}, erasure::{ReadAt, Write}, peer::{LocalPeerS3Client, PeerS3Client}, - store_api::{FileInfo, MakeBucketOptions}, + store_api::{BucketOptions, FileInfo, MakeBucketOptions}, }; use tokio::fs; use tonic::{Request, Response, Status}; @@ -12,7 +12,8 @@ use protos::{ models::{PingBody, PingBodyBuilder}, proto_gen::node_service::{ node_service_server::{NodeService as Node, NodeServiceServer as NodeServer}, - DeleteRequest, DeleteResponse, DeleteVolumeRequest, DeleteVolumeResponse, ListDirRequest, ListDirResponse, + DeleteBucketRequest, DeleteBucketResponse, DeleteRequest, DeleteResponse, DeleteVolumeRequest, DeleteVolumeResponse, + GetBucketInfoRequest, GetBucketInfoResponse, ListBucketRequest, ListBucketResponse, ListDirRequest, ListDirResponse, ListVolumesRequest, ListVolumesResponse, MakeBucketRequest, MakeBucketResponse, MakeVolumeRequest, MakeVolumeResponse, MakeVolumesRequest, MakeVolumesResponse, PingRequest, PingResponse, ReadAllRequest, ReadAllResponse, ReadAtRequest, ReadAtResponse, ReadMultipleRequest, ReadMultipleResponse, ReadVersionRequest, ReadVersionResponse, ReadXlRequest, @@ -79,18 +80,55 @@ impl Node for NodeService { })) } + async fn list_bucket(&self, request: Request) -> Result, Status> { + debug!("list bucket"); + + let request = request.into_inner(); + let options = match serde_json::from_str::(&request.options) { + Ok(options) => options, + Err(err) => { + return Ok(tonic::Response::new(ListBucketResponse { + success: false, + bucket_infos: Vec::new(), + error_info: Some(format!("decode BucketOptions failed: {}", err.to_string())), + })) + } + }; + match self.local_peer.list_bucket(&options).await { + Ok(bucket_infos) => { + let bucket_infos = bucket_infos + .into_iter() + .filter_map(|bucket_info| serde_json::to_string(&bucket_info).ok()) + .collect(); + Ok(tonic::Response::new(ListBucketResponse { + success: true, + bucket_infos, + error_info: None, + })) + } + + Err(err) => Ok(tonic::Response::new(ListBucketResponse { + success: false, + bucket_infos: Vec::new(), + error_info: Some(format!("make failed: {}", err.to_string())), + })), + } + } + async fn make_bucket(&self, request: Request) -> Result, Status> { debug!("make bucket"); - let make_bucket_request = request.into_inner(); - let options = if let Some(opts) = make_bucket_request.options { - MakeBucketOptions { - force_create: opts.force_create, + let request = request.into_inner(); + let options = match serde_json::from_str::(&request.options) { + Ok(options) => options, + Err(err) => { + return Ok(tonic::Response::new(MakeBucketResponse { + success: false, + error_info: Some(format!("decode MakeBucketOptions failed: {}", err.to_string())), + })) } - } else { - MakeBucketOptions::default() }; - match self.local_peer.make_bucket(&make_bucket_request.name, &options).await { + match self.local_peer.make_bucket(&request.name, &options).await { Ok(_) => Ok(tonic::Response::new(MakeBucketResponse { success: true, error_info: None, @@ -102,6 +140,63 @@ impl Node for NodeService { } } + async fn get_bucket_info(&self, request: Request) -> Result, Status> { + debug!("get bucket info"); + + let request = request.into_inner(); + let options = match serde_json::from_str::(&request.options) { + Ok(options) => options, + Err(err) => { + return Ok(tonic::Response::new(GetBucketInfoResponse { + success: false, + bucket_info: String::new(), + error_info: Some(format!("decode BucketOptions failed: {}", err.to_string())), + })) + } + }; + match self.local_peer.get_bucket_info(&request.bucket, &options).await { + Ok(bucket_info) => { + let bucket_info = match serde_json::to_string(&bucket_info) { + Ok(bucket_info) => bucket_info, + Err(err) => { + return Ok(tonic::Response::new(GetBucketInfoResponse { + success: false, + bucket_info: String::new(), + error_info: Some(format!("encode BucketInfo failed: {}", err.to_string())), + })); + } + }; + Ok(tonic::Response::new(GetBucketInfoResponse { + success: true, + bucket_info, + error_info: None, + })) + } + + Err(err) => Ok(tonic::Response::new(GetBucketInfoResponse { + success: false, + bucket_info: String::new(), + error_info: Some(format!("make failed: {}", err.to_string())), + })), + } + } + + async fn delete_bucket(&self, request: Request) -> Result, Status> { + debug!("make bucket"); + + let request = request.into_inner(); + match self.local_peer.delete_bucket(&request.bucket).await { + Ok(_) => Ok(tonic::Response::new(DeleteBucketResponse { + success: true, + error_info: None, + })), + Err(err) => Ok(tonic::Response::new(DeleteBucketResponse { + success: false, + error_info: Some(format!("make failed: {}", err.to_string())), + })), + } + } + async fn read_all(&self, request: Request) -> Result, Status> { let request = request.into_inner(); if let Some(disk) = self.find_disk(&request.disk).await { From 21c414a5e99b3b4bfd705d51ad4bf2fa784771aa Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Fri, 30 Aug 2024 17:44:21 +0800 Subject: [PATCH 08/23] wip-remote Signed-off-by: junxiang Mu <1948535941@qq.com> --- .../generated/flatbuffers_generated/models.rs | 207 ++- .../src/generated/proto_gen/node_service.rs | 1129 +++++++++++++---- ecstore/src/disk/remote.rs | 49 +- 3 files changed, 1037 insertions(+), 348 deletions(-) diff --git a/common/protos/src/generated/flatbuffers_generated/models.rs b/common/protos/src/generated/flatbuffers_generated/models.rs index e4949fdc..aa1f6ae2 100644 --- a/common/protos/src/generated/flatbuffers_generated/models.rs +++ b/common/protos/src/generated/flatbuffers_generated/models.rs @@ -1,9 +1,10 @@ // automatically generated by the FlatBuffers compiler, do not modify + // @generated -use core::cmp::Ordering; use core::mem; +use core::cmp::Ordering; extern crate flatbuffers; use self::flatbuffers::{EndianScalar, Follow}; @@ -11,114 +12,112 @@ use self::flatbuffers::{EndianScalar, Follow}; #[allow(unused_imports, dead_code)] pub mod models { - use core::cmp::Ordering; - use core::mem; + use core::mem; + use core::cmp::Ordering; - extern crate flatbuffers; - use self::flatbuffers::{EndianScalar, Follow}; + extern crate flatbuffers; + use self::flatbuffers::{EndianScalar, Follow}; - pub enum PingBodyOffset {} - #[derive(Copy, Clone, PartialEq)] +pub enum PingBodyOffset {} +#[derive(Copy, Clone, PartialEq)] - pub struct PingBody<'a> { - pub _tab: flatbuffers::Table<'a>, +pub struct PingBody<'a> { + pub _tab: flatbuffers::Table<'a>, +} + +impl<'a> flatbuffers::Follow<'a> for PingBody<'a> { + type Inner = PingBody<'a>; + #[inline] + unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { + Self { _tab: flatbuffers::Table::new(buf, loc) } + } +} + +impl<'a> PingBody<'a> { + pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; + + pub const fn get_fully_qualified_name() -> &'static str { + "models.PingBody" + } + + #[inline] + pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { + PingBody { _tab: table } + } + #[allow(unused_mut)] + pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( + _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, + args: &'args PingBodyArgs<'args> + ) -> flatbuffers::WIPOffset> { + let mut builder = PingBodyBuilder::new(_fbb); + if let Some(x) = args.payload { builder.add_payload(x); } + builder.finish() + } + + + #[inline] + pub fn payload(&self) -> Option> { + // Safety: + // Created from valid Table for this object + // which contains a valid value in this slot + unsafe { self._tab.get::>>(PingBody::VT_PAYLOAD, None)} + } +} + +impl flatbuffers::Verifiable for PingBody<'_> { + #[inline] + fn run_verifier( + v: &mut flatbuffers::Verifier, pos: usize + ) -> Result<(), flatbuffers::InvalidFlatbuffer> { + use self::flatbuffers::Verifiable; + v.visit_table(pos)? + .visit_field::>>("payload", Self::VT_PAYLOAD, false)? + .finish(); + Ok(()) + } +} +pub struct PingBodyArgs<'a> { + pub payload: Option>>, +} +impl<'a> Default for PingBodyArgs<'a> { + #[inline] + fn default() -> Self { + PingBodyArgs { + payload: None, } + } +} - impl<'a> flatbuffers::Follow<'a> for PingBody<'a> { - type Inner = PingBody<'a>; - #[inline] - unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner { - Self { - _tab: flatbuffers::Table::new(buf, loc), - } - } +pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { + fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, + start_: flatbuffers::WIPOffset, +} +impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> { + #[inline] + pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset>) { + self.fbb_.push_slot_always::>(PingBody::VT_PAYLOAD, payload); + } + #[inline] + pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> { + let start = _fbb.start_table(); + PingBodyBuilder { + fbb_: _fbb, + start_: start, } + } + #[inline] + pub fn finish(self) -> flatbuffers::WIPOffset> { + let o = self.fbb_.end_table(self.start_); + flatbuffers::WIPOffset::new(o.value()) + } +} - impl<'a> PingBody<'a> { - pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4; +impl core::fmt::Debug for PingBody<'_> { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + let mut ds = f.debug_struct("PingBody"); + ds.field("payload", &self.payload()); + ds.finish() + } +} +} // pub mod models - pub const fn get_fully_qualified_name() -> &'static str { - "models.PingBody" - } - - #[inline] - pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self { - PingBody { _tab: table } - } - #[allow(unused_mut)] - pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>( - _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>, - args: &'args PingBodyArgs<'args>, - ) -> flatbuffers::WIPOffset> { - let mut builder = PingBodyBuilder::new(_fbb); - if let Some(x) = args.payload { - builder.add_payload(x); - } - builder.finish() - } - - #[inline] - pub fn payload(&self) -> Option> { - // Safety: - // Created from valid Table for this object - // which contains a valid value in this slot - unsafe { - self._tab - .get::>>(PingBody::VT_PAYLOAD, None) - } - } - } - - impl flatbuffers::Verifiable for PingBody<'_> { - #[inline] - fn run_verifier(v: &mut flatbuffers::Verifier, pos: usize) -> Result<(), flatbuffers::InvalidFlatbuffer> { - use self::flatbuffers::Verifiable; - v.visit_table(pos)? - .visit_field::>>("payload", Self::VT_PAYLOAD, false)? - .finish(); - Ok(()) - } - } - pub struct PingBodyArgs<'a> { - pub payload: Option>>, - } - impl<'a> Default for PingBodyArgs<'a> { - #[inline] - fn default() -> Self { - PingBodyArgs { payload: None } - } - } - - pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> { - fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>, - start_: flatbuffers::WIPOffset, - } - impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> { - #[inline] - pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset>) { - self.fbb_ - .push_slot_always::>(PingBody::VT_PAYLOAD, payload); - } - #[inline] - pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> { - let start = _fbb.start_table(); - PingBodyBuilder { - fbb_: _fbb, - start_: start, - } - } - #[inline] - pub fn finish(self) -> flatbuffers::WIPOffset> { - let o = self.fbb_.end_table(self.start_); - flatbuffers::WIPOffset::new(o.value()) - } - } - - impl core::fmt::Debug for PingBody<'_> { - fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { - let mut ds = f.debug_struct("PingBody"); - ds.field("payload", &self.payload()); - ds.finish() - } - } -} // pub mod models diff --git a/common/protos/src/generated/proto_gen/node_service.rs b/common/protos/src/generated/proto_gen/node_service.rs index f72dbb10..5e677b39 100644 --- a/common/protos/src/generated/proto_gen/node_service.rs +++ b/common/protos/src/generated/proto_gen/node_service.rs @@ -454,8 +454,8 @@ pub struct DeleteVolumeResponse { /// Generated client implementations. pub mod node_service_client { #![allow(unused_variables, dead_code, missing_docs, clippy::let_unit_value)] - use tonic::codegen::http::Uri; use tonic::codegen::*; + use tonic::codegen::http::Uri; #[derive(Debug, Clone)] pub struct NodeServiceClient { inner: tonic::client::Grpc, @@ -486,15 +486,22 @@ pub mod node_service_client { let inner = tonic::client::Grpc::with_origin(inner, origin); Self { inner } } - pub fn with_interceptor(inner: T, interceptor: F) -> NodeServiceClient> + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> NodeServiceClient> where F: tonic::service::Interceptor, T::ResponseBody: Default, T: tonic::codegen::Service< http::Request, - Response = http::Response<>::ResponseBody>, + Response = http::Response< + >::ResponseBody, + >, >, - >>::Error: Into + Send + Sync, + , + >>::Error: Into + Send + Sync, { NodeServiceClient::new(InterceptedService::new(inner, interceptor)) } @@ -537,9 +544,16 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Ping"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Ping", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Ping")); @@ -548,13 +562,23 @@ pub mod node_service_client { pub async fn list_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListBucket"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ListBucket", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListBucket")); @@ -563,13 +587,23 @@ pub mod node_service_client { pub async fn make_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeBucket"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/MakeBucket", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeBucket")); @@ -578,13 +612,23 @@ pub mod node_service_client { pub async fn get_bucket_info( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/GetBucketInfo"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/GetBucketInfo", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "GetBucketInfo")); @@ -593,13 +637,23 @@ pub mod node_service_client { pub async fn delete_bucket( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteBucket"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteBucket", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteBucket")); @@ -608,13 +662,23 @@ pub mod node_service_client { pub async fn read_all( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadAll"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadAll", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadAll")); @@ -623,13 +687,23 @@ pub mod node_service_client { pub async fn write_all( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteAll"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/WriteAll", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WriteAll")); @@ -642,9 +716,16 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Delete"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Delete", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Delete")); @@ -653,13 +734,23 @@ pub mod node_service_client { pub async fn rename_file( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameFile"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/RenameFile", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RenameFile")); @@ -672,9 +763,16 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/Write"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/Write", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "Write")); @@ -688,9 +786,16 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadAt"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadAt", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadAt")); @@ -699,13 +804,23 @@ pub mod node_service_client { pub async fn list_dir( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListDir"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ListDir", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListDir")); @@ -714,13 +829,23 @@ pub mod node_service_client { pub async fn walk_dir( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WalkDir"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/WalkDir", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WalkDir")); @@ -729,13 +854,23 @@ pub mod node_service_client { pub async fn rename_data( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/RenameData"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/RenameData", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "RenameData")); @@ -744,13 +879,23 @@ pub mod node_service_client { pub async fn make_volumes( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeVolumes"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/MakeVolumes", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeVolumes")); @@ -759,13 +904,23 @@ pub mod node_service_client { pub async fn make_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/MakeVolume"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/MakeVolume", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "MakeVolume")); @@ -774,13 +929,23 @@ pub mod node_service_client { pub async fn list_volumes( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ListVolumes"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ListVolumes", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ListVolumes")); @@ -789,13 +954,23 @@ pub mod node_service_client { pub async fn stat_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/StatVolume"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/StatVolume", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "StatVolume")); @@ -804,13 +979,23 @@ pub mod node_service_client { pub async fn write_metadata( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/WriteMetadata"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/WriteMetadata", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "WriteMetadata")); @@ -819,13 +1004,23 @@ pub mod node_service_client { pub async fn read_version( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadVersion"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadVersion", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadVersion")); @@ -838,9 +1033,16 @@ pub mod node_service_client { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadXL"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadXL", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadXL")); @@ -849,13 +1051,23 @@ pub mod node_service_client { pub async fn read_multiple( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/ReadMultiple"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/ReadMultiple", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "ReadMultiple")); @@ -864,13 +1076,23 @@ pub mod node_service_client { pub async fn delete_volume( &mut self, request: impl tonic::IntoRequest, - ) -> std::result::Result, tonic::Status> { + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { self.inner .ready() .await - .map_err(|e| tonic::Status::new(tonic::Code::Unknown, format!("Service was not ready: {}", e.into())))?; + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; let codec = tonic::codec::ProstCodec::default(); - let path = http::uri::PathAndQuery::from_static("/node_service.NodeService/DeleteVolume"); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteVolume", + ); let mut req = request.into_request(); req.extensions_mut() .insert(GrpcMethod::new("node_service.NodeService", "DeleteVolume")); @@ -893,19 +1115,31 @@ pub mod node_service_server { async fn list_bucket( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn make_bucket( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn get_bucket_info( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete_bucket( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_all( &self, request: tonic::Request, @@ -913,7 +1147,10 @@ pub mod node_service_server { async fn write_all( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete( &self, request: tonic::Request, @@ -921,7 +1158,10 @@ pub mod node_service_server { async fn rename_file( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn write( &self, request: tonic::Request, @@ -942,31 +1182,52 @@ pub mod node_service_server { async fn rename_data( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn make_volumes( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn make_volume( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn list_volumes( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn stat_volume( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn write_metadata( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_version( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_xl( &self, request: tonic::Request, @@ -974,11 +1235,17 @@ pub mod node_service_server { async fn read_multiple( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn delete_volume( &self, request: tonic::Request, - ) -> std::result::Result, tonic::Status>; + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; } #[derive(Debug)] pub struct NodeServiceServer { @@ -1001,7 +1268,10 @@ pub mod node_service_server { max_encoding_message_size: None, } } - pub fn with_interceptor(inner: T, interceptor: F) -> InterceptedService + pub fn with_interceptor( + inner: T, + interceptor: F, + ) -> InterceptedService where F: tonic::service::Interceptor, { @@ -1045,7 +1315,10 @@ pub mod node_service_server { type Response = http::Response; type Error = std::convert::Infallible; type Future = BoxFuture; - fn poll_ready(&mut self, _cx: &mut Context<'_>) -> Poll> { + fn poll_ready( + &mut self, + _cx: &mut Context<'_>, + ) -> Poll> { Poll::Ready(Ok(())) } fn call(&mut self, req: http::Request) -> Self::Future { @@ -1053,12 +1326,21 @@ pub mod node_service_server { "/node_service.NodeService/Ping" => { #[allow(non_camel_case_types)] struct PingSvc(pub Arc); - impl tonic::server::UnaryService for PingSvc { + impl tonic::server::UnaryService + for PingSvc { type Response = super::PingResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::ping(&inner, request).await }; + let fut = async move { + ::ping(&inner, request).await + }; Box::pin(fut) } } @@ -1071,8 +1353,14 @@ pub mod node_service_server { let method = PingSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1081,12 +1369,23 @@ pub mod node_service_server { "/node_service.NodeService/ListBucket" => { #[allow(non_camel_case_types)] struct ListBucketSvc(pub Arc); - impl tonic::server::UnaryService for ListBucketSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ListBucketSvc { type Response = super::ListBucketResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::list_bucket(&inner, request).await }; + let fut = async move { + ::list_bucket(&inner, request).await + }; Box::pin(fut) } } @@ -1099,8 +1398,14 @@ pub mod node_service_server { let method = ListBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1109,12 +1414,23 @@ pub mod node_service_server { "/node_service.NodeService/MakeBucket" => { #[allow(non_camel_case_types)] struct MakeBucketSvc(pub Arc); - impl tonic::server::UnaryService for MakeBucketSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for MakeBucketSvc { type Response = super::MakeBucketResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::make_bucket(&inner, request).await }; + let fut = async move { + ::make_bucket(&inner, request).await + }; Box::pin(fut) } } @@ -1127,8 +1443,14 @@ pub mod node_service_server { let method = MakeBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1137,12 +1459,23 @@ pub mod node_service_server { "/node_service.NodeService/GetBucketInfo" => { #[allow(non_camel_case_types)] struct GetBucketInfoSvc(pub Arc); - impl tonic::server::UnaryService for GetBucketInfoSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for GetBucketInfoSvc { type Response = super::GetBucketInfoResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::get_bucket_info(&inner, request).await }; + let fut = async move { + ::get_bucket_info(&inner, request).await + }; Box::pin(fut) } } @@ -1155,8 +1488,14 @@ pub mod node_service_server { let method = GetBucketInfoSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1165,12 +1504,23 @@ pub mod node_service_server { "/node_service.NodeService/DeleteBucket" => { #[allow(non_camel_case_types)] struct DeleteBucketSvc(pub Arc); - impl tonic::server::UnaryService for DeleteBucketSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteBucketSvc { type Response = super::DeleteBucketResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_bucket(&inner, request).await }; + let fut = async move { + ::delete_bucket(&inner, request).await + }; Box::pin(fut) } } @@ -1183,8 +1533,14 @@ pub mod node_service_server { let method = DeleteBucketSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1193,12 +1549,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadAll" => { #[allow(non_camel_case_types)] struct ReadAllSvc(pub Arc); - impl tonic::server::UnaryService for ReadAllSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadAllSvc { type Response = super::ReadAllResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_all(&inner, request).await }; + let fut = async move { + ::read_all(&inner, request).await + }; Box::pin(fut) } } @@ -1211,8 +1578,14 @@ pub mod node_service_server { let method = ReadAllSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1221,12 +1594,23 @@ pub mod node_service_server { "/node_service.NodeService/WriteAll" => { #[allow(non_camel_case_types)] struct WriteAllSvc(pub Arc); - impl tonic::server::UnaryService for WriteAllSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for WriteAllSvc { type Response = super::WriteAllResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::write_all(&inner, request).await }; + let fut = async move { + ::write_all(&inner, request).await + }; Box::pin(fut) } } @@ -1239,8 +1623,14 @@ pub mod node_service_server { let method = WriteAllSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1249,12 +1639,23 @@ pub mod node_service_server { "/node_service.NodeService/Delete" => { #[allow(non_camel_case_types)] struct DeleteSvc(pub Arc); - impl tonic::server::UnaryService for DeleteSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteSvc { type Response = super::DeleteResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete(&inner, request).await }; + let fut = async move { + ::delete(&inner, request).await + }; Box::pin(fut) } } @@ -1267,8 +1668,14 @@ pub mod node_service_server { let method = DeleteSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1277,12 +1684,23 @@ pub mod node_service_server { "/node_service.NodeService/RenameFile" => { #[allow(non_camel_case_types)] struct RenameFileSvc(pub Arc); - impl tonic::server::UnaryService for RenameFileSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RenameFileSvc { type Response = super::RenameFileResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::rename_file(&inner, request).await }; + let fut = async move { + ::rename_file(&inner, request).await + }; Box::pin(fut) } } @@ -1295,8 +1713,14 @@ pub mod node_service_server { let method = RenameFileSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1305,12 +1729,21 @@ pub mod node_service_server { "/node_service.NodeService/Write" => { #[allow(non_camel_case_types)] struct WriteSvc(pub Arc); - impl tonic::server::UnaryService for WriteSvc { + impl tonic::server::UnaryService + for WriteSvc { type Response = super::WriteResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::write(&inner, request).await }; + let fut = async move { + ::write(&inner, request).await + }; Box::pin(fut) } } @@ -1323,8 +1756,14 @@ pub mod node_service_server { let method = WriteSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1333,12 +1772,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadAt" => { #[allow(non_camel_case_types)] struct ReadAtSvc(pub Arc); - impl tonic::server::UnaryService for ReadAtSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadAtSvc { type Response = super::ReadAtResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_at(&inner, request).await }; + let fut = async move { + ::read_at(&inner, request).await + }; Box::pin(fut) } } @@ -1351,8 +1801,14 @@ pub mod node_service_server { let method = ReadAtSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1361,12 +1817,23 @@ pub mod node_service_server { "/node_service.NodeService/ListDir" => { #[allow(non_camel_case_types)] struct ListDirSvc(pub Arc); - impl tonic::server::UnaryService for ListDirSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ListDirSvc { type Response = super::ListDirResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::list_dir(&inner, request).await }; + let fut = async move { + ::list_dir(&inner, request).await + }; Box::pin(fut) } } @@ -1379,8 +1846,14 @@ pub mod node_service_server { let method = ListDirSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1389,12 +1862,23 @@ pub mod node_service_server { "/node_service.NodeService/WalkDir" => { #[allow(non_camel_case_types)] struct WalkDirSvc(pub Arc); - impl tonic::server::UnaryService for WalkDirSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for WalkDirSvc { type Response = super::WalkDirResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::walk_dir(&inner, request).await }; + let fut = async move { + ::walk_dir(&inner, request).await + }; Box::pin(fut) } } @@ -1407,8 +1891,14 @@ pub mod node_service_server { let method = WalkDirSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1417,12 +1907,23 @@ pub mod node_service_server { "/node_service.NodeService/RenameData" => { #[allow(non_camel_case_types)] struct RenameDataSvc(pub Arc); - impl tonic::server::UnaryService for RenameDataSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for RenameDataSvc { type Response = super::RenameDataResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::rename_data(&inner, request).await }; + let fut = async move { + ::rename_data(&inner, request).await + }; Box::pin(fut) } } @@ -1435,8 +1936,14 @@ pub mod node_service_server { let method = RenameDataSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1445,12 +1952,23 @@ pub mod node_service_server { "/node_service.NodeService/MakeVolumes" => { #[allow(non_camel_case_types)] struct MakeVolumesSvc(pub Arc); - impl tonic::server::UnaryService for MakeVolumesSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for MakeVolumesSvc { type Response = super::MakeVolumesResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::make_volumes(&inner, request).await }; + let fut = async move { + ::make_volumes(&inner, request).await + }; Box::pin(fut) } } @@ -1463,8 +1981,14 @@ pub mod node_service_server { let method = MakeVolumesSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1473,12 +1997,23 @@ pub mod node_service_server { "/node_service.NodeService/MakeVolume" => { #[allow(non_camel_case_types)] struct MakeVolumeSvc(pub Arc); - impl tonic::server::UnaryService for MakeVolumeSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for MakeVolumeSvc { type Response = super::MakeVolumeResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::make_volume(&inner, request).await }; + let fut = async move { + ::make_volume(&inner, request).await + }; Box::pin(fut) } } @@ -1491,8 +2026,14 @@ pub mod node_service_server { let method = MakeVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1501,12 +2042,23 @@ pub mod node_service_server { "/node_service.NodeService/ListVolumes" => { #[allow(non_camel_case_types)] struct ListVolumesSvc(pub Arc); - impl tonic::server::UnaryService for ListVolumesSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ListVolumesSvc { type Response = super::ListVolumesResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::list_volumes(&inner, request).await }; + let fut = async move { + ::list_volumes(&inner, request).await + }; Box::pin(fut) } } @@ -1519,8 +2071,14 @@ pub mod node_service_server { let method = ListVolumesSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1529,12 +2087,23 @@ pub mod node_service_server { "/node_service.NodeService/StatVolume" => { #[allow(non_camel_case_types)] struct StatVolumeSvc(pub Arc); - impl tonic::server::UnaryService for StatVolumeSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for StatVolumeSvc { type Response = super::StatVolumeResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::stat_volume(&inner, request).await }; + let fut = async move { + ::stat_volume(&inner, request).await + }; Box::pin(fut) } } @@ -1547,8 +2116,14 @@ pub mod node_service_server { let method = StatVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1557,12 +2132,23 @@ pub mod node_service_server { "/node_service.NodeService/WriteMetadata" => { #[allow(non_camel_case_types)] struct WriteMetadataSvc(pub Arc); - impl tonic::server::UnaryService for WriteMetadataSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for WriteMetadataSvc { type Response = super::WriteMetadataResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::write_metadata(&inner, request).await }; + let fut = async move { + ::write_metadata(&inner, request).await + }; Box::pin(fut) } } @@ -1575,8 +2161,14 @@ pub mod node_service_server { let method = WriteMetadataSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1585,12 +2177,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadVersion" => { #[allow(non_camel_case_types)] struct ReadVersionSvc(pub Arc); - impl tonic::server::UnaryService for ReadVersionSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadVersionSvc { type Response = super::ReadVersionResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_version(&inner, request).await }; + let fut = async move { + ::read_version(&inner, request).await + }; Box::pin(fut) } } @@ -1603,8 +2206,14 @@ pub mod node_service_server { let method = ReadVersionSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1613,12 +2222,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadXL" => { #[allow(non_camel_case_types)] struct ReadXLSvc(pub Arc); - impl tonic::server::UnaryService for ReadXLSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadXLSvc { type Response = super::ReadXlResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_xl(&inner, request).await }; + let fut = async move { + ::read_xl(&inner, request).await + }; Box::pin(fut) } } @@ -1631,8 +2251,14 @@ pub mod node_service_server { let method = ReadXLSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1641,12 +2267,23 @@ pub mod node_service_server { "/node_service.NodeService/ReadMultiple" => { #[allow(non_camel_case_types)] struct ReadMultipleSvc(pub Arc); - impl tonic::server::UnaryService for ReadMultipleSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for ReadMultipleSvc { type Response = super::ReadMultipleResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::read_multiple(&inner, request).await }; + let fut = async move { + ::read_multiple(&inner, request).await + }; Box::pin(fut) } } @@ -1659,8 +2296,14 @@ pub mod node_service_server { let method = ReadMultipleSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; @@ -1669,12 +2312,23 @@ pub mod node_service_server { "/node_service.NodeService/DeleteVolume" => { #[allow(non_camel_case_types)] struct DeleteVolumeSvc(pub Arc); - impl tonic::server::UnaryService for DeleteVolumeSvc { + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteVolumeSvc { type Response = super::DeleteVolumeResponse; - type Future = BoxFuture, tonic::Status>; - fn call(&mut self, request: tonic::Request) -> Self::Future { + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { let inner = Arc::clone(&self.0); - let fut = async move { ::delete_volume(&inner, request).await }; + let fut = async move { + ::delete_volume(&inner, request).await + }; Box::pin(fut) } } @@ -1687,21 +2341,34 @@ pub mod node_service_server { let method = DeleteVolumeSvc(inner); let codec = tonic::codec::ProstCodec::default(); let mut grpc = tonic::server::Grpc::new(codec) - .apply_compression_config(accept_compression_encodings, send_compression_encodings) - .apply_max_message_size_config(max_decoding_message_size, max_encoding_message_size); + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); let res = grpc.unary(method, req).await; Ok(res) }; Box::pin(fut) } - _ => Box::pin(async move { - Ok(http::Response::builder() - .status(200) - .header("grpc-status", tonic::Code::Unimplemented as i32) - .header(http::header::CONTENT_TYPE, tonic::metadata::GRPC_CONTENT_TYPE) - .body(empty_body()) - .unwrap()) - }), + _ => { + Box::pin(async move { + Ok( + http::Response::builder() + .status(200) + .header("grpc-status", tonic::Code::Unimplemented as i32) + .header( + http::header::CONTENT_TYPE, + tonic::metadata::GRPC_CONTENT_TYPE, + ) + .body(empty_body()) + .unwrap(), + ) + }) + } } } } diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index d1803d21..dc1e9996 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -1,4 +1,4 @@ -use std::{path::PathBuf, time::Duration}; +use std::{path::PathBuf, sync::RwLock, time::Duration}; use bytes::Bytes; use protos::{ @@ -16,6 +16,7 @@ use tonic::{ Request, }; use tower::timeout::Timeout; +use tracing::info; use uuid::Uuid; use crate::{ @@ -30,29 +31,51 @@ use super::{ #[derive(Debug)] pub struct RemoteDisk { - channel: Channel, + channel: RwLock>, + url: url::Url, pub root: PathBuf, } impl RemoteDisk { pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result { - let connector = tonic_Endpoint::from_shared(format!( - "{}://{}{}", - ep.url.scheme(), - ep.url.host_str().unwrap(), - ep.url.port().unwrap() - )) - .unwrap(); - let channel = tokio::runtime::Runtime::new().unwrap().block_on(connector.connect()).unwrap(); - let root = fs::canonicalize(ep.url.path()).await?; - Ok(Self { channel, root }) + Ok(Self { + channel: RwLock::new(None), + url: ep.url.clone(), + root + }) } fn get_client(&self) -> NodeServiceClient> { + let channel = { + let read_lock = self.channel.read().unwrap(); + + if let Some(ref channel) = *read_lock { + channel.clone() + } else { + let addr = format!( + "{}://{}:{}", + self.url.scheme(), + self.url.host_str().unwrap(), + self.url.port().unwrap() + ); + info!("disk url: {:?}", addr); + let connector = tonic_Endpoint::from_shared(addr).unwrap(); + + let new_channel = tokio::runtime::Runtime::new() + .unwrap() + .block_on(connector.connect()) + .unwrap(); + + *self.channel.write().unwrap() = Some(new_channel.clone()); + + new_channel + } + }; + node_service_time_out_client( - self.channel.clone(), + channel, Duration::new(30, 0), // TODO: use config setting DEFAULT_GRPC_SERVER_MESSAGE_LEN, // grpc_enable_gzip, From df8cdfeaefa4a8a8e88303c6d28422aa9a9a80b3 Mon Sep 17 00:00:00 2001 From: loverustfs <155562731+loverustfs@users.noreply.github.com> Date: Sun, 1 Sep 2024 22:38:35 +0800 Subject: [PATCH 09/23] Update TODO.md --- TODO.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/TODO.md b/TODO.md index 973bef0a..2a9e5ba5 100644 --- a/TODO.md +++ b/TODO.md @@ -34,11 +34,22 @@ - [ ] 对象锁 - [ ] 复制 CopyObject - [ ] 详情 HeadObject + - [ ] 对象预先签名（get、put、head、post） + ## 扩展功能 - -- [ ] 版本控制 -- [ ] 对象锁 -- [ ] 修复 +- [ ] 用户管理 +- [ ] Policy管理 +- [ ] AK/SK分配管理 +- [ ] data scanner统计和对象修复 - [ ] 桶配额 - [ ] 桶只读 +- [ ] 桶复制 +- [ ] 桶事件通知 +- [ ] 桶公开、桶私有 +- [ ] 对象生命周期管理 +- [ ] prometheus对接 +- [ ] 日志收集和日志外发 +- [ ] 对象压缩 +- [ ] STS +- [ ] 分层（阿里云、腾讯云、S3远程对接） From 32b64e74d05dc7c59b46e123ea4dec8d57fa7410 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 10 Sep 2024 15:32:44 +0800 Subject: [PATCH 10/23] rm zip --- rustfs-inner.zip | Bin 116269 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 rustfs-inner.zip diff --git a/rustfs-inner.zip b/rustfs-inner.zip deleted file mode 100644 index f5aad0ed9712e4fb3394ae9d99d56e92e6513d1f..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 116269 zcmd42WpEtLwk_B#W=4y_7L&zjF*7rx#mrKRnJs3PEM{hAW(JFyS>x~FdH0@~n27mt zrzPKhqoqO$7t1@!QOMydR0AT(emq92RfdBD9YXJb@0K@;?}2LfiuZfWN;706M(KZ16|o|8#Z=mgu@DEN#UAp>Xt7>#}w7 zNmA0&rB+B&+a(mgZd7F(#gQk*gckh%R;v(1nwn}H)UvjR@5p@X`+DWuWkvWSa-zbk zTg}5~b#rDe`1s*rDCF~A&VB>xbnF zzV(@R<_(|NPWRPy-;bv?_7z^6#z&%$L+H;n+s|tvLCYP1g$- zM}{sUuLsxLnUBS6F=VZ2*A2psn zUac*KuT!7;-qS-d>E{zrF?a8eTTc;JOGAg;@8}YOELlQ%!@-WQ0zWCWsqTc=PeX%K zwT=uqF)UQH8wA-xtT_JMyxQszD$jdv4wHTC`o2BvJWi#ztxa71z08){-59+PXL&{K zt)N}88^KMN-i}YlG}3zPNK)`POB~%^43AP&_kedn5T~N6M=hfuf?o9b*y)m9QlDe}Ry<0p!NLXgVn4d;j9%O=rWmW0du8`-COB~bzR@l^ zp!UMtWG{Vw)mpX(Sv=L!GRm-29$-X(NyaA}Vtvt(5M`~(0rX0idgG<{U!y4kHuf^@tv4o3jx@YVAU3JGhFu8dt^{`k$_wylW zVSb2{^`ZSn^9yDFwgFg=gBGd#i9jOk&H@FGO7B9KEa!Skc zBUCdq?^jVnTInQ2m3W&9U(J>9*@j44>JYZdB`QOuc-7%@?IULjj>s|5y&CEi1-)M% zriRjoh+>Gwhf3^kbnNw%Kjd|h7qr(RzDf}qGzc=(Ntkr2uqG_G;+HYSiG`l5U;oUR zY!n;C=XqdWV@b<&{S#vPk!-wmel*0sQwL~lq+govAUVSmw9KF3h}kXBVqP)Wo@w0! zEIgQyH=pbJ!;YvRO%z$3#|!X%G#}yx)=bQW!8{mgXen_bIuz~uRowjI#xxx(Vt4xz zHEP$-1Wj2f$GJcsw>>2T@Ah%VJa;z9d)DaXDAvB#YVr8--0r=4NhaJ@Z7;Vr5!b(b zupr~uJWG=yX(2S+N5NC0q?<0 zp|uTh>rFy$3Y4oufU|0D2o7_%v8r5~a8~=Z8Ly@kdkrx;rRF?LO05{22PUuwNlQC) znYQ#B4p__-Ma-qz7CAUN>_)c_agYUP0a#(d1qzOXzh9VjYFQAGGtzaB128hM!_uyelxf$oBsExd3QA{-0&l_e^ z)!I8Vw9fXke>}ahjyKeppO|YdoQg5~+0m%N7Wgp>J`FNsx4Ja^37R9=@HDNi9bQoB zmtt&2g(5XM76mW3=3En$`pNtFhGge+?)}rwKmRMT`3fn_=r+u+MV@L6)H05Qx@vnN zjfWhtkhvYDav}@kwd5nywvC&m^aq~7Ff-wP<}xvB+wUB(y>?QSueomUe7><&xr@Tm z<8gQy@PUpfkqNzx(ssq&mF|S;GA(FskFcKF)AdAhwHxwy3be7Z?XB;r}a6Xplp$t4QcK29d)y3TK5gOJuoO!;_}x6zcCkl} z%3nT3FTaGH{%XI1+-zBEuKQ5nXnF2v(O!49EScEl<*kgHaht~+a~Dxwm{#4M_@27k zE%KcyhZ1mw=$7BZ0|v7`OigbmUJMZJxVzJHS$jPXJB1^9U&Up1a8b_Y`t?m~+IYz! z7<_E|HLQnr7nBR$^)-P6na3XO&nIF(S&zh2Tcuu4rI!@4*e+yqdBZIpYSD#2Oxw^L z)(J3lU5$fju8#BuX5nPxnVR2jqY+7U#R$@j30#ZIv~Z5%b3Ti`>BQ;N%^Uc`s#6E> zL#)f$x@n@={?Y_XGbVl+k-N54z zY(kcB+W@n)(;001Mn@eILxgiHg1z-r{0>tY5DD&zP6C{z8&X4w7AO3I!!Gb#cl+x` zj=G-+t!MjMumn#wtoO*>hqdHOqh@hSU<0_Q`M@~g-ra65>TWN7HJKj7d*ml z(gpCc!xVs9L>VnOnmSPQo=h5xn+z3?A!Z9?Vk|PJ}ooodZvYUxa z6JxG9ZOQL3)Mi3_kyY|v-4`h9yu&F7-EZ#DuaZBo2VSqQ3ZSW7T`9uEC(H^P3~a%* z1pDp4_sPjkYdr#m&m@T&C{TVI^1423kkqnFUuBRz^Y;}t67Nzqe_p-LA$uJ?`|kx# z_1BG3IYUirubpVx(II~K6J(?V31UsxBV_{+p^!j`(O@8@mX_XpJ9dI2=xy5YCTm?I zqDJ3QoAq;MsF>PQY(tYRucV)ShKi<6<SG3C(xG|neBz#u$DoMTt@V6X*Ux>QEnc1L-k&!nfbAsMFrJUo>Nc<#@s!wL4*#8;)I4j-#1 zk|#q4=m(e#;H6u?1+k-}2#1?u0{_gH5?g^w0)_TxO;7ff<^ zm@AewbCxzsv#*4(Eb}u>RPLUoL>@%Dv#PFY_#zpfpR5Q3)XXS@1<(dqG8DzlfAIAj zRpYPukOr%0H`U@8dOP^(fb~mk-%Eb~jf^54g8&y`v77^aSR~>Fy{kh0g`Y4G1v!+L zh#V5x?bC@Fw#Jo1O6!paTJK$~%cBr|S>QKTbwwnE%-!}RQgc0KzPr{4V4c+{;Abur z07*E#Ln#XK8P1x|xP#{%(Kq+ivxOIdUz37NjA7Vy=dc)C@AYLjV$bLO+TQW={wRpT zgU4rQAX!g$vI)BNr54T}x5Z3883|13OS^6GtIP`Z0nE%VE3eA1F#Kvqpq!eK9~hzy zJfv}i)Ke|bC9H2WXC<*aQQ8MvMGqvSyeeuSJ|3r;DLr2kpa61HiOxSVAr^1Ccz$tv z%0NO2wvO4lrIlEJ%EHctoGr*teB~#!(y(AC7iQov2;VI|68Y-R_{uR>wg+i1O+nye z6zfAKXfxvgrK$#ng#3!rOj8DO8z;-AJ5kF8^P`RPu%LLS9kSS3s5@ReR+QI0qCp*L zh=Ej^D(O^8p?kG^2wdEQKNK>dZbk(bKfuh*)=Wn5EfMKLHhLR7ALmK%6o)X@ZK5`a ztgWMU-?<)4p}?597l|?@!(?ovka`9j>02WlkER@PP$et$VRX~YFFET7@hf@9Dtd^l z_CP=JQ*Ag{U~`@N@BGy>w>lheNQyC$Hqs?xQ7a1jGtXC zU4AE)LERJQ2O@j-B07WENUl=WD}d-*GKyH1a#>0^V*Fmq`h2s)S|M$hBre2c@OwBp zRsJ`}#!j`yr%nEqpO8Nxo#VOmG_RTSZ^xW_e@I15&Bm7<4*CfQ{6eWgfK?YO1#Gea z>Xi&|(A^E#L4wOMdB^jE9f;Y6KP>{Y7|bq^;KV)(hTuicU(+7ZEly90oq+9meFfjU zo9AZ;_(Nn=BD^_j9*feAN#!d6mKePGWoa(arKfQZyHp$4t}cR24}6GMkoG~eZvqBx zWuY?HtQvD%LLiRvW=aH7J#^HorU_k-JyRixdie`4bq<#dOvMkM8KHJgP*eZ(-Ms6a zo%zz?N@A+66rZ`L1|jhBX-wD`&3017*)GD!U%%UoS@=DQ;xeihCdMz#3^ zUoy!RgdKF~Q;zU$8TUQcoAdRe8ey27K5%*5l+Ffxhz)W4i+qLfrKpPGj-i+~e)n(9 z&ZX{dPlnL%-KA2)Rt| z1mO{sZVZTNtJ2!zh{gRM8}fhu?88VW{TOiLqF*+8B3O?hbi{3lYldExc~?u=m_!Ymw+#yv78V2l}rnS?|0UtRMDt^ zlq%=Sue!k=_c`^YmBH0X)6RNnYL8rW|1lL2%(LhS^V4un6G=9W0+ZIybBP!h6=j%7 z>Y>2}yjD_vK!ZdNi7`n;Y03RO8fYdU?g@_ygF1s(&4&TMSg|HXJ;03EyR4&5v_ z0M!qVCC1lFGyd9m%U|QZI%^_fyV{t|CSI-=3J$BT6gH!;w+!ikCK*Ync~b2Q%fAxD zKe#l!icdPinoXxlA>4iO?AV&U+bP`k*q=cLce-R~Ho>x(rBoXRQbajVFoKR-VFh|L zXt(nEXF|+e4jek20ddh9+u+{`;?a@8ZZ7Eblk26XLCVTKx;zZ-6! zK>Yp_{y1~fOVY3R2<~~Vson0nXgPOzd}99H!jYI#VYXL&;&(uIU*2~+T#PzXxO~Zz zju19}J5*A5Vs!)PwpyFk*i@J{315Yy9A(aQ4 z_b*AxqC(}TnSw5wMRC#%NIB5T(4iBLFtev+WU@AZt!3%~En!hu7`)qAq6`%|$ty`q zzIORkc*7rjGw<#cX*z4Ef=dB3jV^e+>~M4}=v0ch(Y1tBk>Te0{51IOsM{bHsRFE_ zD@`Iky{&6q=UJ<*%bLRpwIcx?Lu58qwWGhMf1)Zlg!%L~Gxh^4OSTQEe6q?M42 zFhVb)U3I6oJLdpnzAD6kBj<5QGOU2i%i)V34qeb+akR(3E;TN*mX#8PBA4VRPEF4K z;7Mq4f%F6%eCJaGOsj0$y>n->r@V`DUoevS@r^9M+AH6~*^H+4bwd#56{Q3_!^$Go zd@~Q=Tg>l)j4VNiK1hI4(%G6Aqa}V}9C(4>cwm5*7|Jn!*L~H-!6?+2MSC{=`WUKD zn$QMn7LoHnG0I8nZB(4^!CuzGvdnb5fnRki{u5-z+G?4tUKn4OL;`(of>KPOpdfm_ z+mN+DYEJztC}bP(3%berySiY6H%4tIy@^=E820e}fokXJc+u^2T`LaE6Fz6dg;K~6 zo(~MXewz7@U{&9S3~fm^AA)0;vmS1F{2{_gMksfD!NMNdQ0c4k(Nj| z9d84I!EJ0DhuXBrwD@_oRi!1On|&vU-~6Hz(wZS0-u*J$KLD3Un*+7u2j}~Q8V>40 zW*$-1A2lXRoi^-0!l9$oad@4W^lWWOykLUWSooY`@ZWK^!TY>exH&$5IFpU_@@-AO z?;VKZ&QX1+cfoR)QDdQz`50`)OXM~cd4Z3f$1`n$NV%*?XurrdK%EZ*!|ts2MNK+u z3?hmo_Ma|ya9R)ADJ?<&mvk2JdNdOk;rq4gFOwfy5S z9pB2XWdv}G1=fNMWag)6*%WTP-_>Al5JZSLtT`& zWMuy*`}Ua77=jo$f>Nyhcl`K+K7y;%Y*)t1sLfZ-xGQ6dC5HTKAsX%Un-8qq_=TZf z)HNv#MN@15i;)RfzqAexe(zVSl7w7L80S)=+XmmG@*SEAf>lc=6(!uz6Q$`rW8k6n z;z6ABbn|!T&Xmd!b7GcIpyZVoWXj@f)pOwUSoJUDS0EJyx>IzUWVB>{w3$R_d2i>z z#r4rw{e1XW-#5>ti!I{7fnSPJjB3n+;du*=84X(CiyIA;Y-O32tr3<`X(guBjXME` z3o56^$V-Rn7jX9)c&vXE3Y}0(uagBH zlFOdT>x$I&m^Z=)u&i{4~LEX1%((^8Ffh;&qdkl zQXr=B)TVF^KTlgwSCR2DXu5qsuX=C%UCwuE zHSWioDVJIJaJ8OhN~K+f-?nTtnt%2StrBBOfa^5nd8A9X&GYu40&2_m)X8O`%{6#8 zCvDxvvN(BY;aMd-j6#-+m$ZnLXw4hS^()6L0drs&^VEzSA5iaJJ4P4sS>0L3GVjhG z<*t~yMqb{;pLo|l`D2&|z{{}jMR9be=BiUII9sv)KARH^vUlQ9e_6s4W?%}mE3-?F zGpc$bcN#zf?tIfCBWIdEg}G#|4GEZ7aIdJCV?zwI8tGPG zLYQl3@EJ$8O%LSP-qI}@<1$z7ba|}^`VQt?-50EIi#05%MI#BLod#`nHtzv7ITat? zh^e|ng`ECVGGO75L8?0WX0;6>5NUFGhf!#95fU%s-lzb-V=3RMU9x&6C&9l#27+%v z>TUxs_vG}gWq^cSbG2aZ+sK~q^$Uh(JEDm&LH$p-fy#T;ZOctzhAb+QJ6ORViIo9n zT40a;SZOTcbu~W6R+r_fyOHLVS-;N;M!^wB)7BQrbxFju{U`6n>CIb_YkOu5p^J(O z(4V_!o#4FKrLNx1jD{BEC^Sd7eoTq3%>w>Z@n?P+YpCwSZf52wjK1HYP2EA2w1;&- zpz~=oDb0HTIO=r0&j5Wn&VP*3_0ISqy>1%XL^W_BZz~*D!z)*4wCgn3O#s-T?H7zS zxWGN}s_cIYmnFFCiR8TK1TQYfg<%R5eYIc<0^|T2i8y~+SRvZkBw5e3fjQk+XCBB; z{rK!_8iA4IVh>m-_P2;Qru^1>&Xqa77Jd+V!k8QI(XtPP$)vE7lboS{&i3#MMm*6wJ6>L*3ulq6Cd zq#KGZr_3HT{`2$vj6mK?rT~lQ7Z26jmIu=>9TWPbUl!Gon2fq$)&{}M;yg~zN=*(m zY#FEPYKg-ID9Tte=VY>m7j}!DRUI>-<8=EX{ksPBGCTH1I58cP8^H`r6H9h6w;+f^zPPUE+ z$jVSLdYDiXu4z`=%5Ayuvj(>v%eg}d8U`Id8eLx@3YDIxnM1Ie7Y`X{M(JL6&ZL|j zb)}tA`}Qh0C&A{V?grNy^zl5n7xPN-@*%+M>SkuY`7!K1XnpT*2{pVB9<7Q|-2{id zT>;GkAx92DPmA4cMEl}bWL;Jr4WpJ)tl9bOIE3yuC#LpU)PMRI>q(FR8Pl!MEiD}J zELvL0WgUw*Qe4t~e))SqZ{_{nY>E`5^~&$Kyw7o;mCjU% zb(KOGDTHGTRsma)4opMXf>EK7YbF-f=;D$-{On2a69ZF3-p#s4 zm@im{i>~S$moCws9Pl>8qYEZVe_^77If+sxjhqlxb${V`_d70-S)srG$T=7U6bKH zj~1zHj^PTEVhubRrsIg_q^>QuR(>*(1Pd+V#((i@d#gFBs)OCyL853f%xHk9tgLQq z{Njut7d3EPu(00XQ~bMLu}U>x*v+1 zlu_f88D(dNzP~Tnn zH29i!*G>0dNuF*sp<0VacbX{DiKB1Trz<9XV1QGDOhOc$g9vpWJnUf^Pkk2zuF<6(WoKCX^#N)CTC0!vzyrj!qm??f(!!Rz- zgA4gOw50)iA9r|fUY1SEMo=kIDRyR*v-}DWbLlwZonbS75vSl>E>imDSlZFEA5ufx zQ4DGQgVKbf{fOV!$CiEs17)&6S8KuXp1(+;jGaGqB}O1s$zpO4_&qP#`$bzotKK|c z3u0f9)7^P{zlKmP_Dk!uZPAQMF~945nTC_n{lV3)SEFv^!L?<%tmIOyQ|3-$kLD<& z3Uf$fZ?amZqEf;XeALlU{z`KHx>slMS~|np`BJ)?zy}_EXe{;B{N!qpt7u~$qEnpn zO*3?NUFH43&Y%>nyYYGVUom(jH1`!=sgeQvwG*>A+vd+|q$M#22|p$sdX&rZjwbN* zOj+p+gj>o+k$=i7G|D8jh#KwTDnT^u(4}mB>*(^nAl5C^b~5bY3cWr#{XFkim_ogr zfttZ4BraOc{H+(-fE$fXl4e#1(qAtvoZU#w;C@FFk~msW#)Hh^=iE_an&zJpM+uAO zxYAeCyCM}xB;WG#;>OBabIUB4TNB=`60E5dm7-@gPEg@Crd*R3Bg_~Y=& z5mJRh2ZY9d-8(h8b@!{Yg7;EywF9(1sO5@fectg()QcuD4F6cAEI@F5*%RM+;?~nsk9*f}Cvp>NmCK zcBMK@dBZ8q{%cP-5=9Xu#8+dp!*14TBAQr$&wm^2(BD-9<5g-wsL~5%?4OlQ1%u`XCSK zV9;wCEgS-A^ews8GZxya0^7CDB;{@F-i`nuOPc6tRpJT@>WYVcc{275WxVpGSIQ3a zR}OQO*TF_-cw=z^F&f;=47yvDjxn!8Y#JV)B7m%ryr$wjsg#|5%b>yu zs!h8VUa(T%EiM}l0S93$Jq}SUlMl&JAm2A}r|?7dvm+8m#4rbu&R8g_?T z5gRdd1n4gX*sbTBAGaLKtaHf`*vTc#e^vFr?D>wP=H<$}7E}}mKKu!9k}v1z{aluO z(_i^G0h1#tK04jvKWLUxeH8k3#DB=?Twy5-%chIi+(xAA^gEFM;n_aEO3jymX{8Bk zjEaWCwPR8DD4WcyID2z7g1&riT?OUTm4%;7+Y}5P9}!L<=Oecdu9uO^fM{@QVw(xh zJb+G9IPm_i{1yjHaC#OHEJQUm%D<>;^+P-GMlcmIwo|tyof($Y3bO0I4?RBLT47~dS4Ub z?AimmZt6-y;*y04Yc0)+ol9ed4sm`pTRJntzw$?GlL7!B5;;;(qjI=RQANPKoo6T3 z=_aznYaSCcN5kzjj}q$vi5Me=_@N$Es&f z_8Mu8|$JUkv`ag3LRVK8is68g=W>Fm?B0Uj;Ku*)?Wxm0DdK86E!I3 zav@jD?|l5Vt8V4i*Q|WMXa2d4_TeO;;?9^9JOLhJ-weZDLp>M%wWv#^J-#c z^e5ifY_z@U=$8#)qznxu5d}mpb%+K4bgczT-%>SC#r50urMncL-%q3W>=I?aK3WF+ zU@Lt^f}f#jwkDk|^Mgd0R$o9X>*$|J|0PzKB(|z&5ts9=1$`h$gH{aje(UHVb+s@; zHHJp8a)WEG2U`DmyJ;JwxtYm2v<1R)SeV(cqC=#z>M)a<4}N$7BWdd1NM-q!2ZWt8K_9Y%>So?}vFo%eVWe@O0iBZ5y|8V6fR5z=* z<7N9$2;(EIl!$g-gQ$B|THvrbMZVcIlY|x2uNe#vK?|+~dKN0}aZeK7G8Ftyhw z$XF}^4os*Obxc66_f-ZMNFGYndA8|19(7w5ee4l4*_A@<+vUb!p3L?VVMk}szpnl^ zW658Kgy-1iv#Nx3<2APSq_v~Sr22y}6h37KA&I-12n%VIKbF7S5GVHT(m+(#oCqGWNZR$Fm@@C(E_LUqb0JHjRDBfixC$}MWsH=rl(J|!;Fh%~!!T3?n@3%?BN(KXL!O34 z4%f0?kZ%k&1%>ee?^aaXCvp$B9N)kyME0>AE4B5S2P3;e#r8sJ;T9 z_k?+i^$y&0yzevTvBhQ!mdO_WrQtK%rqQOgdD+4k7=`;J%Z+1DF1!fQTw53g2&6dtOOcPCJ9lg)4kb zRaOd}xX7a3Z0PJ@>;67{$kW5^t=7BMeR-rQ>aE4+p~R`&dQod`Cmc0ddj=P4mwK7L zEhhj55Qd_hYs{spULpgY<0Q5|%Q-wIooBh_fZYP>{F)TV1QqGj=@C-<=Dc?lNyaoxVOY5T|QfQZK)HpP{ib)`Na{ zXjas}V6GsLjg=cC1K95WNrHHkF9`|6S8SpESK$keXZ-L$6rUqXDgLcE{fIEEMA!K3FqhCl1~E+Ri$0Ac zj>_x^`vFz~V@YLohnfJ79k51!of{!BhR71|OTPyIdn++O4O;xtaOt)EfTnWl>tMpC ze?LR^kFCQ61`ZNR6SVWm%ax#Ya?Fl*%6{YUr8e1|xSu{}-|>}MJd_j*{EuL=8X3pl zM8f5Sxj*82DR4H<&zst^U%fx#+Jrr5d*^Ro$#`@^Px10C?dmoW^}!Dm8mPHK*;`zM z+t-0)KMA%?TjZ|YKrK-!8b{P*+BA1kO3NSw zTbv>FzJn7BeNQDBUe`yZeDs3o2m!<#=*<<&4&O|=Oc~+@OQu1?c8@7j5oMB3ewV!A z86QgwrITzJ)1{!BR{DX*>IC(UgPV*CLh`cFR%mK0fsKSblN`~ED6fpQt^cPc>z~5f zG>s}xZ-W~^N|VP<1N&+ZnEHZXxtk5~@3j0)S{@K-LLB9s3IwB@S09p|acS8dt)jMG z*JvmzGGrMee%eC=c@|vB{Xs{r(sDJhTZm483thr2d7hA#Z&2U8%b4<$_uE#wY5?+9 z#U*MCyg9O#_+}z8OF7Ll6%;;%8+(Rx!Oie$N=_e>q|Ie@d;p5qyG&T8<+HvR&-I2o zEH1GQH^hBA|J;%h2C|;%_k^QFBhF&5kcu!QuRGp+J&ZqaZD{1pv7)kwgjOkV_f&Ho z$oZ%8)p4QQk+}-|?wfd5eeqY((dW@@44bn|$dQ#@%upFT{#5&UJ%X5sHH^S{n zzcB*@q5mf{fRR3?k(s`s{*{ch>J_qTs5}TP76+UH1rKOXf)el)fg1`IO+q3d|4n=d zf%rgFsW%h_TV`@JKOHf}r(z(oXk87P z(|N}z=VgeXA;jMjvkR6_b%g72>(oGDB$Zo2hmlJaE^C3)2@>mOZRIaD9fd_$hB7CL z6RvR9)4Cx*vMPii-@2QWUNnt1oAM#?5ok}DjbmN2kg1G$K2J#i3F;y12f=J0{26-B zE|2t-@@mNsS-zT|a6ZJ=0Bn)*X2gmXTC?`d zK8vF3Fh;R$?gR$+BM5p#4K4O(jL%{?VIybeAtDSxPOnfWzpSFOtmvixL3Df(h zN_Jzj(XxM?A^+6k-&g7X7GD3UVE@JN`inEb-M=_v{<(+w+ZQHAj!w1?CJg@#T(o~d z^-rAt23-F;{eKL6OF=-l0ne%y-!*85DT2}S^8Q0IAEN| zuC4KSVG-=H?yBA+K48Wj@*_xgdE-yidsz6UnDdhq-_ZV)n-Cto8Y)1RDNRj2i9%ap zE$X9V!La^arHt5gQvTSaBL4#-#0Uw12m9o1Bfn)W>$wW$M_WFm#~^E? z_Z|KyDMkmMSaCdsK>?psx=-PnKo6GP#EKleP7F3LN7twe>{eEbG+WXOD;kRFI!fEI zg&Clz&mX%z3e{PUN!&b9sT`b98R@QDpt|oUp9~6^!VBAPh%?Bw-E}og!WRMk&Y2G4 zb0aYl|81C`sXIu3k(PA=@)n5Hf3e_QVp>RDnj@<*Fr(PFjwEHV%lH^8+&mZsZ+r1B zb8Y#8&=lm3CZ2j?*{dm4^lR3K60N3`$FG1+;)B^rO`DtgIRWLp)QBOYebxTgw9};8g+d#uW$SpS2D26+eQ%@FgGK00Z~IO-1}9h7 z{7>E<;vd2rA&r@l6j{fP*V)D4+IL1nmD{SqYW8y;MojXC57_@uh=CF`i2g5q{DZmw z-9!KCQTwNd2>!K){wFN|=acF`0rS7Oo&KSYeHnnDLW4&53gRFPYlppp3}G0)l^MN08zz>qn=|M zf1-txCSt%9_~jX{^M6EM?)r@CZM&PAx|%&qnCX(IFl?@6x^Hd`Nl1sGuSBMOi^?PZ zK7q*Qg_yml3G0O?jT0qvOZtIoD+((`>V@6f$ttV9V?9p#Ge*YIHsFU-=+o;eeR_B% z6JA=;vQ667o^K*DV4h{m&ifB$-JUh*@#i*k>loL5RQD}8E!K(;4_sbNV&1zA&vj-~ zURZbQ;?JNFP_=~k2$WYd&KX=}hoehK`vS!&gx&qNvn|JU*qaNLfkE#!o|RjemAr4L z?>5I9r`h8x8g-5|zNqeoxOmvBNfGv{ur;_ zxy((`EgXoxFbOuUegWWXIUtM!rleEmoeo)kYjyc){q5mvo*uXX-V)Mdvcp%)`C8_Euu(U@qdDNwPI}jZl!{J4M#Z z;`F%CY8F7`r3juem04XHZ)htq=pR40n)yEa`ku3i__|)#R`%AWF5itmHWQxPVx-m- zq=%JoX3DT0jbfKXcBY2yAcOBWYF2z7lqAhhQ`5HS`JZ)%-c`4{2V;5>@aF8 z_aoNcLiXD3kb?qvx6L&>li?_VM~>v2vk`_-s%Oger$a{TuCgJJcjY%T7q`-Fv!aK? zlXDaV?2pFR46AC7LE3JmyI>UV$;F4Mm@Bo<9x;z>{-R}fEZLq{p!br_wcKGAx5vK1 zp%~T^%D?_5%iz$gvqLyg_^c09T0x9}Z?RZssd(3XH_;UO9?U3EcAE97Lo|0tW+Xl;-8@|2N~NWYK0 zZr+(e#puL3(t6Rr_n@z{8f4Z{5J9Z;1GUPc$d)T$@GqRTAhAwT*P2gtAQFZ?~m}xp`_8M>%bv?_>sBw zvj4PWt$?1~9VpR0V3Ja}X^HDZm1EeUH!Ke!lOcsKMR4Iv4w1%aRN8xREEyHw(#~Co zqRFBHW4K#}r{#oS3u~P0fS({5;9PnkGKW@=zJBK_V8BoBW{W;gSFCAIxA zy1*H17&IrUD=)xfHHJh+*4&1^Vh~D*MG+tZrX^%5!~A=eg}0ft?Y=F)%NOQ^`{icx zx(<2ZH9z3T&3*1S z>sEHij<{oxF2`+kK2Lh?vE^*{ma++YZ?E%-Ogx({oT8B~jG?}^8$QG!128_}#5m=8 z$4{&za`Stnf9a&sevxEsx$jr3$h}bPv%wB1PC8N|{k?7{S%S^Jvi=$Vheeh1Gw+yZ z<&z;5solfpli`-Ti_^!&)#Wq)lT1nxeY}^=NfAzRfIfcet*$LK)~xFVfRED;sgRj~ zi0bWgtHhzO4A%NvE{V4yqC!ebc*~g&TAl>ns)Wv}NUc@pbDo4VU|ql3VE<+OHdmL* zogwKfF?t>(nKO%Enj=l6gLrIBQ+dZMO=0`AiWWBQ6Xr=d_^W~idH3_8X3+vJj{0^y1w5S$$HMMn5e&d#B;w;os4CvNTRi9X^%;~SF-29j( z3-&w2P0=vQ2%w=Oc?&$4SYd_zq^p|rI4tM0eY%3GD0cNO)UD7nAG7*_)nr0yn=Lcr z;-0;JK}v_kj%21*BUQ@SZ=Pl=D8|a9XhTL_rp9-00OKTLe>ISW=c3wqImnjsg(b1p zwFJq!%YhKi@h{FM&3@W!&GIol$1JsdP&6D)DQG;LSJnu`P4I z=W4`)SDAt5*}Jy1$pck;wnsYkbjy-XHC>-)ysBR=dgkjDMvKt(d22co9(+i21R2Q^ zx)TiNZjb}WG+3F|?eaC(f?vfMZCEPRwWp7hOz&T*%(nCV!6-3i58=-zz=Syiv=~02 zLGah}(LTVH#Ps>dYKuEkZQ`A(SmU4Z+$qD%0s`uvhwZ=z4elD zOkVYXXvTWOM@*S2X1zYfdcN`@aRE$Ta)_GtQbgTQ%Q4bUPfrL;%fQf6vN_CWi0E;@ z5K@(?WZ_>2E7uEImuK$`zI}8AT@gHQSiGzc!!~Y9?IiM;fx5kgs`KpFM>{rpc)0v|D6IdFO z6#k?Llw_MssjPOb*kEN5e7?-o!CQj>LkwBOUfXl`+G<~H5U5Tw6Wo+muq?#&Oi|V@ zBJaa#9SZx?$X1q5Tyny>Y!bv+6QN`V9JxTH#yOH!gf6b3X zj1&)4PqC=~Z+DGvSA$Xt)xz*|}CD{TyvUsYit~5(LQ|*7_4y^m{ zxSIsUkN9x;q#jP6;~9^w-nd`;iYuKpNhF~0kX;1Y##0nVd$o-=X_ecF^2)t~aZ({7 z`?>V?ExFQ9{0y}Oq#e0Sh;4*1s`Ml?a4}dbo#Qw_$-r^7F&9eN(4Fk-uZ~-OR&eq` zi(fQUH>1brZ~R}(y;G2A&6X`(wr$(CZQHh8wac~3wr$(CZ5z96ZW6U`+blHw)O!@H-#o1Ds!?$#=u=vOi+;7B7oZ25^g`J)+3BQ`^ zBf&CiPK^=uhS;r0-B(|H_t?``!(JJ+{{LWdyU+0sGXCz9|C!Fqh*^6E&o@fE{Ktf6te z2%$XEjgel2DC<<6G|C_S-m1PGU)6(8vOMWT4p!S-Z3_ zXfg|w?o|BK+Z-i9VNQbSS6CP`S!Gs57bt!6S1cs&FOA45s^t$Q6=Q^1@IFo0#@dUng7O6d z7HMinE6XQ4vx9_+!-8ScPS$w04#OKdUsa>$7BkIsz!^1@vAyr*BG3o>;IhKApe$cG z>*X5ncgKw#C5Mdo&iLW=Tx90x**IH9+dF^f%@_uBoN%JHL4 zc>=yp(|$_r=FIg|$?X~efkm>wtZWH=!U(5J|L_%;;{HhV2I&r2k#Cc6>qFDXxv8P+ zwmer6mXuZ6*>6DpA;<_G>z#oN%|d<3{xbwO_*a=J_c_bY(wnKu-zRr~Dj(w{!0?#! zare6Eajue5Zs>C?wix1X1%L&@lW(Ya>s@K_H|QJVfJQxfaXXUz(BToJaytmr83ht7 zogUjNq6S4Xl(C*Uq%>*MJAsZigrd|E6Mfk3puc6ZiV3b`D@=A6h-p%~f=P1v`{E@) z;Q@aTX-*nw*$B9*NipU@8slV|!3y(^xaz|4%Lvjh0i_ciF2{)Flo!t1KTLN+KQAT(yo zN~pb0V#K8z2%Kgm+x9cGf-_Glx=`r)@O)t)*2zfCu}df=U)Tr}pgNW)k4egLx}Ojl z$f&K;$yKzV__xebD<(Ctg6>9L5ekNS8wZYLf^rO=WfNXXex0OZmZa(Ydskg|u4Ty2-6Kd_d%vmoPv zn$ZWx363kxdcbniTYx5kC`(RlBA%=S+eb+C4DGRl2Itb43lHe@;_geW8_(H!fN@h% z<6|CX_;AdPv%WZ=Sm##jLOLsy{&r!#ee2iiTWg@wsG~d4v?7J{ejY_9oXu({re>9C zl9lYs3ShJBXYY4bm(qkBue#ri$!8@4*&~_k%}KCs ze-eU=u=j(y2K=24`2~_3_g%0!JEj$eE9MbbKU_qI-5{3+KCmQ}@z8O+x#yoplDe}; znUBI*fF_2j7Ag@=gTzQ&x!g3whdDUXOeYlhXMMsuNI+Fn%Sd$G7wnR)*i&sljgnz= z@`>i?m;24xInvM?DjC$-(r&hGiS13#n4DL6%Hsq9IIO<4Z)5^wajjZ z`dw{uh*G#PViI$?62q|qz<8hqXfB$SO)hoCv6<{n7a_lu;IArS(__@g5Tl$XKE6t& z>e}35>&~XM3ffr$=ZeOJK%|6<5hvSolvlUmkYF9ZB#qbMOr($&6)X#`lV_>6A)7St zelIV#(X^muf8)?lRUbetiY}P@{<|SD{3)Bh5`O}xQleIt(98Uy1SF*m6Al(r4<(`s zES8{~mMWD~j@iKMZ#J)|LV(y?f(uHHDxlEoaaGXZjH~3JHP}N(ESlw8S`YFPt0=u+ zIM;a!YhkwRX^YxunQEmP%v8ojakMIF4Rc1((5O;$0fL}*P(p}d_Zsqrs5os3#INB* znxlsbeo0yYBkQGt#jVc%i$n?(CJfMlv&94;Y2vQUQTJn~;oSX~y%-hPoa?!_74@ zVr%_p4flbmXbL9;~70I%Sx;wa@-s+tEl8vnh4A6n8;a{-n0E1cAen zSwfCKTgsT5cGf}wwwHUL+rI=48-OQkmnHx*@33O?z199{)!-QG?1J*8ISrLabqAA2 zqOISAf* z{QpB=!TI+P`ad51KUrm=viq+pi;1PPwX?pBp{Kp8%l`#tyVA9F-V{stuF+>)(QDeH zO+lj6R3n%yS2FJOp+N(I5v>vg027epN&oTb{AK3fZy*rRM7rrN7>pL~ zdGY7ZwP#0Z01fm^;1+%R=^c-}^K7SIw4d%XH+_m@JKwJMifW1}3c3PwOkvgD?G?YM zWB&s}gx|?6>HXoEUwfB+NDg|D<5Uon6tcywzAvNS61O~>t_&r#IFLuh)WoP$%sps*>e6P{JJ3w z`PkPM%+BkxpH`mHer9|Q77Xi?De4T%-k3?3HFE+nhhVkVCr=crH)As zvE8)J1lz@(jH-8Q{GPW+`~{cG)b94(t?0B3ZH29x14zCXuWzpVj1@1|N+lKTo9j3J zUr(sNCJ)mu!RbFB&MW?m^oJnA9Kvjgp)zXt*mskEt0sZ{VWP|q0tzX}_LT_W6lU2Fbv8HPSd--IV)*na(g%}T*se(f) z#}_^HLWi_;#)dZby#l9>)$R9cTi%B8MKvOf4z373I_8f3()gS)r#A%1`_S!qCzVI1 zho{?>)vMRc&=#Xn$22cGWw3q!rN`JcrM`&iLU0V-Ntgop>lW#Fl$7$G$Ew_!Dnn9` zvcx++ncCa{LS<&sG-oev#@f)%q0CZH#{Jn(*3LhO(5*q|wrC3YF5BCYiHLHHAj}9YKaD z56-1_S_NaDL6MOe{eF@S!>}^efQ{x9Zl!@>Wj+GN8g)fJ7mH_M5;bJ*o1d$vR=Ka>*wG86KP`OUFs4b zYFXi}l9u#pgpWw*xL$;N$yS18f4~+M9}%-u=+Y^o0;QJM00TqYXJ!=dSTHSwi5~X| z`2`Cshnx$a>1D~r(*<1=xYCU28_~{?3&R88w=r>1FxWd?U`m2CiE+ElC-Xy|BQPKg z7tElF${2+K1+DWx1{x)ck8Jyf-8k0!2`0WVZWy*Mrb&e)zz>S)`sx^`ac1oXVG-q) zaVMPg4J?rHa~wd2aMWXo3_)LrMYL)3i7$Ig4toM!1+tX>Z27q$S^IV)-uqh+7kC!e zuv;Ys`u>rhGvJe8X`j94()?V-dTZP+wVyPR z#E;ENL||HnA+F6yIcrSD!&H=C+$ZRlXlo2xiPS}IP{1zd*r7vpQ*ijQxgz=ETgfP; z4j^mk2Kp_+RjUnnukksn{*^)aLq-D&s2xp}ys<+uw*^8Ky1zOX{MG}#Qnd?gSo(93 zISX7>x*+`K`L!k`w@DKX>5}C|CV+lJRmaF%qe(BU$W?*^u$VV8cc|vCw0!-EXcoZ!|JJc zRAzmP?-JQ4N!K4BB3+>VV(k6yY`c^4ud@PCtT94TDUsC0%_0E)vBGw5kzF%xZ!A9jp3HjtxEE>%AD%a;0x}O$vwF<|{_YX<09#T&_-L8kUn^$c z5t-*vlF4L*aWw+w%ru^vfv5%#`|-%k6tS*w*&UPbxwegpiCnd|@}+ejzN{YXqV>|L z+V-cosaQ@It2To#BA@=f)@*o#$(@_&>P~UhZ-O&}6dyP+nhJ#6s!O#N42Wq6f#(0r7I{obAhO>8rRK0F6;VY9ibD=9 zGL$0pFR0_P(A7B%e=P_Hm?-L?h0XTZuIR&xr{plic7;na<4DSq)Ym!&1<0|yWC7A=5jtkEl|(C7-Vf@<1t5v>>lsEHpUFn)X0!E}h? z!g}g&eWfqLoiu_Bz__Ed35l!YtUSHelJg+eyDlW?T;0SP01EM55?Kf)bV^UnuxCk* zrE4j(8li3#<_;iq)En*$(jnO3B!h+fhB{jWt~MI?g;+<+c(u&WhD}ct=N6+`14P&y zXv0JN6vWFff^>shUbj=BW9SyA^=bCG@JWa6l*}J(;oJC+r*vywf8Ea zxuqWj{HDx_%+xhONTN=C>!;=!#o*(Ipm}gAI8=XgE(Om3&|XL}uj;j;nOo6*2cD*z zaSc!}03+FvnP0#h#62Ixm7U4FFBhq980AO|0stkMH{V21F#dgmvC;}b6=PHkan7EM z%`G|~muP`~@8za;QB5{~zQ)pb!P^~d>0hi9fG?Wxl9=|99IGcwHK0As0bSXjuvWsl z{#!24l!onDVu%q*sXBqpg097prw@=EhL;;tsW9ifE@KPwlK|X!De6rntZ7h9b{tRv zgp8=_i`IPKbEf?Y7;xB$s;8&c(ARoRXR2)U51Ws6E{d5sk=R`orq3_b^cX0 z9WsDE!E>_zGOEm#+PvJc7*Gu4iK>oJfYXNEZ#Xg{9?2(NY%qAvxwbH*b+47R1`~?K zF4=i3uIX$+K#mgwOuH|hp*8D_14dwK1MKi(LWLoyJ)u3vx>C>|({{|uDq={wMx*{v z)ZMTUDSjzUz)41~F?nF0b+|QmULF0572x);Qdlj9d^|v2x!=4J`FgDbOh& z!k6VN-1`aYg|UfD2g7KlL|hC*5~ z-{ChpyII=M)od!CXfvfaDjuCaNL9Nm>T21IwL)cHToU4nOkt_kQ1|*wh9jsHr484( z452+Y^8_->Cu1rK0toHP`t5yv2Yq3L(P$!>bQMZPq5zsyyLXF|6}^ii5;j5flCyEK z%qDu!Ms|D@Qd$1eKGlog_Li`I=Ywc0t`^5YM}J`eK;E-adPN8EZ}t0%ATI7Adz$=_ z+zjH`jhx8$hGf}u;>yvJ&)u$7)_N?N*_nb}7ds)gD`MxE5MgoQs{yPqs>Rr+YA@v_ zu7&IIX!)=rH#6;Do^7?w0yd~Jb3qN-hr-fTD%>~=oh$OeA_ZCuoo)=@jA@s&3vr4R zM|!u3POPbzvy_F(G>pgi@1B@%cNg%u0q$J}TAx8&^|JHmA?aOL67&cgn6n{^W-?zw zO)PemS%G5STB^%-8?f)WbdC|os8Rj+%8`<*#j1EdT|pK6Z?p{SYZn*AmRP-6Zf?gQ zoBqVZ=kaNM=k* zn6PfoEXvsS8B1-y)R1Z6BDD@g?7F*v2b7kLsFRNkt{7gDv}3uPxV~G$&IOkj>tIEe zj?vE;WZpu};arO5nNSiP*ZJ;`eXr_Q|5B3P*Q<1$AwIbVN<-fRVl8fSy67I~3#?x_ zlDQRb?`1K#s`2?j^goxqOuR+M1;zR-&I89u>96MaAB-xEso!q+3g^ucrUrUoc?|K{g)Te(Z^`^_QP~z zW~%84F5b*yL6Vn)#>Mlx;w&aVKj%5SskK8?j=N{ZY|UZo2Jdx2wq2{m;S9w-YH-=y9U9Lu;z72JMr^MCWA9Bd6HJ^%E;Y0qrgDl>zL)U(AXdVpq z{eH7ohI42H`LP$Q=nvZ4a3EOnBb=4pyVJt^`;VNn{l}uyA$i-nHKyYdU;b+nad96; zrCm*}Zrx29NkKeMH{L8<*_>dU*p<$_gT@^HG(5=bPaY=j_jehe)s9mo{)U1WzJ5zA zTwa4`8`PL+DBgGsJ_NbkoCK$$Dc!@(!-j_1NfxecS}q5-t;K7>l28KZCj{`GUQ4Wl`Us zcn8#v>^H?I@Q#-qc#N;=b>M_krppd`PO}A>!%~A%Ok_5+LHZI=98etm&Bl=qY*s9s zDrg&N;7OPCyZ*IRGVNLh9|Yt3R~tP=Cw`Xnt#SPeTiUK&NtNyH ziAwX0%PTQt`pJF(h+rNpg1a+DlJIFWsH$!lyg2eMJTx~MnKzut(u{-qeWjJj7l??3 z&lT?%$$owsPoO4_BnW-e)rg1tbMuqXG-aM>Pa`t<$x!ercueoLTNgPI&)-CnKSFr3 z?404@-cM~UG^$+&C~w}0Qrn25GS%ev9Z0V0!s!#p7(2NP$*Gp@JHyv~(9+UQ@}u~XzkVdne>WRk zpYp9edP_^T2w!LH2&<4Hx_CaMkD;BncHXbY(ke79ziD;`8Y*)=OC2hT1%A0UB&_ib zRfd=xc)X-Qi>)%dwZu=3(CfG@-F*{u;T@H9tRO!W3qAF1AV+n}0seyT+F5?;w{BJF zAq$iGU>tf|rn~(!egZmD>R-@#QuW-t>f#MK@4+#&`Rh|Y0f7Pbrz#pKpm7D8yQLB3 z896Mcb1HF6^VEr-#^2+L?6=1g8hIDZdxH0nhuBg?H*OJR@e)Z1dx z9KaE zkH#HtR1YPt{@lGEZOZr8J=98c>Z``>vsup}OsDB-tO9hd_GK5v8(9EqPKqyZ8`QY> z_wPQd)N1R&E%(I+`U>6NVi@KrjhWgz!d<`p3p-(N!i$uSWz2>|X_ebgFlMlG8z*am zKh*uQuQ1bWZIZd$tNVz4*+Qhdp01e(#JPyPx{|T$BKH$m+jf7#hBv*$p(K7dXDmQ546e@9O{cpq-cVtGCC-?(DI9bb^d1O#R6t{vQwL4ObCB;^7bOnB@I zH#1YH^zh3E9e)zdIL&ef+=-t$$q+^@hO5h+)MAbTej*q#R7CA#YQObO4jZ_f1!tMi z1(NylVgm4kU68o=j_EZl8((cA8!2ekV|+j%WKJu$IK42V)qu^k%4hjh>nS4OQ!#Pn z?XGfE6x90J-e3Dvv$uD`|2R*4->Tc@7FZPT;xBq^*KeYtxYF}>BcFeOXsLM(=3XBQ z-2Aztt`(p#z~}Nz)>GX5u-D7p=p;Ev5Oa!o>y4FamkqFf!2GJ4K9&-X`BgYBGF*qmG%ChGc*$?qZ9b}%uB%T{X<-)FDZ7 z@`K7zER-i^n(f}hLG@%pfb-{&;dtFWDo4NRWAXz)x5o@D1{?pZgkhV`pKaHdYoE5B zQ%?Mv7Ym+-)BHiOpn#q4=D+h*Cq~$sJCgB#=SDerOHx zD`?(OS%=1_eN0A){c;Da&c#aH=AS_$SO@VsrK^CnJO>Pb>TsnUM9Za63UFW^bxvC@Sv;;{_N)W{D(@P|vot)Gp_15PUJfMKlzCMYi@B z&{Sr&ajI)ae?YanNZU2-!oJ&?Rurq#41VOo4T@9%qEU`&?Stc4z`WQ%2r31kBEpc?29-n>1 zxnH;~Xb~0(#9zpbC=G}tHAes(&%mQaST(;;xrD4!^oOKxEel~>a45#0$uc5mzX8}& zMRe108t!&k6JI-Rw%<6#^?*`6n<2wFcXaE<0DG}7(dw`T(mw1lpOlpVcv)^O_e|rs zK&$(?k!=fhMv2yd6cDpG4Ll7#ECe$kHxrT(#G2k0YPCdvr*ZaQNuB1;*fZ#YUGBO& zwScwXE>hQg=P6%X7_AQQCJweMb7=Pl`jL0MfiyYs@#B1HM)wO$ zPdfh_2M3CI!xe)!ZKk zRXYPsr3r<~!IUGf*OISO0J*P}+4;DU>%RY`!}*zb#_bITNhdMiUo}&O&u@a%aOpRo zp0OP3If5Zq!I`RhHgmPtD}VhDOkk_AH)(t zx`@C>){a1sPvG}zi+NrJ%&OBW?HJC2omAzB1qj9?2Q}81H0E&t=hIJ?-zW&NGBUv- zhi^ppz8PPZps;vP2!GwW#fk<%tuLX~yW2SK2iY$vh1Mlm2*+jFH)>Iz(n(N-0MX@P zJgZbPI&h(gV6RAeBf1T2)Dv%%wuon*>7J95ty!VsP{A))0)uCRX{`>s7lMlA2k?tI za?zj6xTTF3irAM~NY&a^2hKu-v76@s^EJfEDJw*UgvB$O?~9KK414E6B<-8y5q$JS zuq<*TauE0?Sq$p4+1%CT4cLJC-pb;FTC$Em=+U5Mk{C#*%3u!zTpiyD;zoRvP|ZMq zP#;dJ)CiULTLg?iW1NPw(vYbf0qdcSeOv^~Qszyutng$m+vcr~5hu|phhRQ$N zsQxaYOEO<>Dl%16u0BK=WE1FeWQ6})giF|ZZ%OFbjX1iSPKWYrp{`Yz$sv6DzBX}d zk|?sPkoC+0X@!s{WL5;@sJ%@f)LdYUnoi}8IY#lCtT2xz{0%AK=55;zct}&L|2;rn z>o#K~sot{SS%0e~kX{j=k5pJ0qK4=E7HoNu`N2(JBs9d(*?pj4U=$%qLOe~3w} zy*55Q$rC{2leB>cx5I!Y=Z0uMK-v!QJ6g$k9N;hMT|P6$M%scR*m|O}YUj_c2hc>YstKEXh;spEJ#cDwfn2K(Potv<1`J~wJ zs2{>29*wdy$o?g?46$nM0MgMhVSXwPAtB$ng|bV`LzM}+oeGC7tN#lMGz8-<7J%7a zlhDixXZKm9+1#}Wqow#dv)i&c+t~a#G(9+KI?c-Y^0xjK_fh8FMz!4(I@zH&3}}wLf$$3d|R}xjUu(Y?10K?YA#yG1e@x)&zbFz z*cEaDx2!-T{4^=j z36_AK2y$aLx?>$U+;{WaBGLmWIaW^xJbHPh3P*MY+biN#L3+S)91%6KvPcmb!dd9{Wr6&;^X3xXGon7M(! z_G21{G_oWk$#*wepmPmLg0x?XHK8i%uq|4s0gjbN>Q28Aw)m?&-Jm3M(0Wi>g!B6e z5>e`iwS3Nb8KI$u7j=jC7i&(SXtq?xcXhDvb?eUm`k$ng)at6*YU=IFek95Hut1@I z%LwkAL`;KN;icLEa4fqO?P#9?V`)Fm8+i{k@_yaTdkVO$M=5$|cmm>Q<9D=nk?QX) zX0m693$C|VZXo^`mA{fsE>drKAc*7*4^YGOe$9fBz4UR2ip@8A+@TFt-h8^~>Kd2D z$jwwCX7mtuw2Xdvy&>;e)h5PO8+r|=xa*RN$A6%T_T*nvz0Y_$jK3lg%$7hKr)4j~ zp-z#59PpAJif|6o_=RY)KbCYf_cs&@PD4>A66YV17JHg$(S(lrExa;m+&2$W#PpaN zVvt}a6(K;e`d@`!Zt3f!cd|H~S9(DoqfeksoIq+YIEh8gB7xOT$x`&vtq?>D4o!69 zA4ns;bArKHxd*E(!gCwQ`a1|UY7P*Df3St;RDm|C#P6z7YU&mCg73?ccc))PjbHJQ7Se_I8{g!SwiZ4>yX3& zWU!Sz2ndtdAXWhp^X@=o-L~_X5~fpD=B#E=Ruqa(R#1@hwws%RT+m?y%9)-7YR;eQ zNN~MNvQap*CCIeBC5dcX3d-R_ZJoO40v8 zU@w&>lp?FJG!*rimsYnLQc(9;wyA~vT>V=i*@sG3s7&gyQabjmSRr&m4!wI;zH@b$ zDG(^1=Z$t8EOrA}+B+3O4oI}h7_~@K4|ctNTeSs~I!LryAUG+%fZqQj z!j{P2l+ko7LAoRou%NON!vM~XoKb*-2*w`G0i|4eBqK0ktJo%v4S$11cPcd!1?}Uq zXpK*{eJusX#(TncX(DP-Pi0*b*7TTO`*cNwUWXHu2IWFU^ng|$7McG-SjD|?0GblC z)27hZi_$9ovDxI~Ib*W?Q}t5il%^~$Lx%5F0DDhO2B**&o@J(RgE%SRZ9Vy89-qq` zAGdet&Z1s!>#Om8I8Uk5#i7@;Q6sx^ul|HVQ*%@r{09U_5t8e-A+XuT!}o>a8(1~L z3EypTQNfk{j={9@{_n$B&auBgwU1_=g=(%ikpG_$(5k;}QPHcvMTY;Au z-uS@q`Im6oi^ouOg{^Kwi&p_}h#IzRtwFns&AXgZyt8aALf@&ZtvH!KPPs+q$lx}; z-0tn=nKC3p9ipkIQpsoyRVx{K3FmX$h@@|2O`13VeB38HMv3_DpL*Ss|AI%>bFrt; zQco#(yI6>z?z-byqgIM^P#x3lsL)suOt z-)83eyKywCMKvR;_>f)owZpxA&07s?7Ch%6C^qe^ci}G^WD2{}KRYtXotYeWVJ|K5 zfTvl-V>Rl(g2NrV8yvUrel%dglN|xyYB{<%`eu@VKc7?3FGh2IQng}Dsxnfk^MuvT z+uT|224i87t|M2rf`%eW+@pjTqCwNB2B~&;mycEm$%<0MDb7k)C7%A2q0Z{OA7;$z zo`7P4DloYz17vj-vS!O-daCl5;UA||ZiUCb1InSO?_@o+5hH_cXvp#NzQGyTXH--mH!Xzeww_opTG_Ub!Mu?({OkD$jMz0v9$lWd@DYb$Z zp&D@2ShgOIPn-(5SoS>m#^MS&c3q-E-|X+x>HhNi+b|%Olqsf>B$@X56mAUsns#I> zH-jSWb68m`tc)gh6`Q{RFfLDxr_K6dxK zmsByG#CwC)#|V#ORd(n zYMWTGEiWfp!1>Rhw!rebP_msU8LH6h9#;y-&4aTO*5bGxS0N~6AV8!)Et3werls96 z?Ot<5)G#P6#H;aAHHElw)*I#5nfJxs*0O z)hyCo#GvCR0a%;^Ff|NUbR8^G9+PqHSh;bom14CJm-&SUT%6t7iZm#<88=G{K^5Sl2T4!# zN=L3X78c%eP~1jLgr8(0sxeN=Dk(ALOImn9ls%VV@QVQux*JJB_2aF>ctPZz1Zk7}#CuBxl- z_18cA&b|D(q5-{I?;f+&@kHKkA*sEM-IfEByX&7U`^S~6@77g_ctPO-LOG?*cNv2wk&B;q>D1M_vV;fYZhLN7;q_s8%Q*cMqZ6KP>Q zNN2oLX!>}R*&XJ1dP|X4Cjdb_^}3Y8=JKUs?jkTBYDcDz!!k0rH>aiVATTe|<1v4g z3Pb?D;lxN<;|6^l6BM0990hI zo1dbUI@7HN^^BNxxXKKs^z{V8{;_hfczH=Rb13PCoq0^b&a$)*V(vZ$GvC<-Wv`WquK|Q}$X>c&}yQ2>tuIeOmT^egDhx&CbBe zox;+#k$R%9o9VXj+y3&dbq#XwUgtbF0r-(>bY9xL#MyTG*LNRQ?nfJAw|9KV3cY?3 zdOGzQH079k8u$zUoWfxw;TI)G(>Ki?7 zni)IZo9UI775Hvccb(hwl{o8j8m{>6xxxP6$dJ(d30vbYTQT*DGL8)307D>|1yfxf(tqH1X zCoc$(ui{9x91;lc=DZ|}-d*N}U(Z_iU5}T^diK;^2l1T{`Ym7hXjJ|b&bc^XCvQxP zt!&Xrx1i0Pu(OI=)Dk(P^BBmsAED1K@38|H9$t0JEmH$1GVQbGsd&?{8B$b_6IMEn zd0y5;zRU!SqV^UK4yyC1Xdj4iJNWh$kG-AXy?(tP(;r+ee^1C1baC*?!M3v0O?|dF4x@8@0om(03cR7T!n92F{sw zlqc2D!g8MO?po7iQ_-G}+jMC7bxd?fTN5*gof=_{dNyEs7*oV+z~H#Rq06n=UH7LL zKbzTJ-RO!_?{h66j~MhwgE)r|)$dSvw%?#KRUWiIIHDU3YI3$cph=934@ z#a1D*_&#bh?6hdzS`9J&`+M`Kn7#%v(**2lCc# zFrh?%j$ZQ3hftUUvc@-ENp)cQ18uL-20W#cLJrv)z_~`CRiqbESmFhc(Yq^)HPCBo zf^K6q!f)Gy6^DnCloTyeF7k<8<>MR8-jwC~=Q1CM#&b5ibh%FUb(!V*;2dtNdc!R< z1+Z?x@-Hwilag)7OWZ0r~(#)af1UeFbb|U zr2c{c4%DPwb{g3`2eM1M!|1GTkBdY!zTfk;4UHh$(JKbKR$~>Uj9&Ty5ODrDA z5S#LN#7!m=ZVD?5kl3wG<6+|2&$ucsAf-G3MHJEZIW$r2qjG^rzM?!(=dTW7*m1+1WSSOr<1apMf28(+w=n+v>kw{c>>@x}qW<=A|GB8VfYq zuKAvXGk71-KaJV+JvgU@(te3Fj>Q({&PTx?-<;N9qfthoU_PBHGIe8@8Z(Zac5M{5 zXRVNv6#ieY#=HqC4iFbGTA^){A%C4hy(g!HBuJNGj*pSI>;W zH1B&OESW6*|MQ$<8FJw4y=xdvT_|IFUtMgHoH5z++YG*RcY4bqoBL=Af3Vr`FnrWwi3G2Qa7 znFz$hBgw5LXtEH2aKmWS#IGhz6pbQzn5x2obzJ%sb8)tx9CAU2SvIzRj0tATI1&rh zq^jF$;$f-3S|HH?r@v@&-|E?C|Jnt#LsuqA8bX5zotO%a9D`Kh0_d>km&)0jDAZ1^ zX$s$NOgp}^1{AJ#7EbYSxZP$3PctD#5dplGnjc@CQk9AeESexU)h@KsE}zOZxLhAQ zq9}-6&q$c&)y2OI&Pk`Y{3`p&@~M%PESB;y!_KiPoRrOc6;urTOxENz#tlP7%hS%x zr|cWXFp2zmX$FjGHIl(k+WMo98gm}tY??`CDTcNu{wl{D$^gPV$rBf`$jkw(tpUUN ztL)=;5}@xC99n~b2#YnO4fbqzRy#R|xsl=_J%}IMYJBtoWr&f_neXnyCX`RE2Nw#54q6z9j$VOk$5Le~Yx$xQtnMUl3+}X7UU@5K^E~d6(u%~4 z`5Ye14%17O-d5cf(0J9w4rfcBHA)zSOVk!xY0WMRdnr9P~g!Dza?AN+qG9F@Rpil_+ilJA`BSNOhBh-9m6VQyD5wgQI z4f=vu6G)Q9!&s?6R&TY*7Pc+1eJeIAUYJCa`65bPFj1Y0rV0%W8err+6HE-lg)RX( z*jRs$1!i1dD2zWzyBHFEx^p$Np@7%IZ-g5&rVYV-)x;o;J4H2Ts4=g=3Rg6y>VReN z8u}#^bi=ESU%r;ZPL8w4Og{$XAKH7foAydT?|ve@hpsq1OE?sSk)4wPI`)U=q~9bQ zj3!wfa991$xyk%9k&6cQ;C@QSN{MuTF;C=}zYWi50mjM=N7fN5(vu6(b`L0Hxn|m@ zC8M==x7Gf1Y$@gCunnEHVGbg9G@pNd>TA=8oi|qlp!b@4{F&7@FmyTua;jrEWg|69 z4J0cL+m$eSF<&XNUPhu~T23EY@RjSIVya;xYD?>`C9A;|5zC_KdFzpyK&5$a@#3cL zV&eJZmKHMs;URkf3lUEiu(lC0z%o#$#zkHl(Sq|)h7w2+J_)TXxFjoqZmYv``mg$( zfCe1@d^+%=x)`H_r@LqKlKR?-zUhpYO@l-nWB;TEEWM`$niw zCtlKi(#^yav;+W1Gn68To-&|GXo!75tkF6Tk6B%J{H_~eKf;xD^c_&d>_#yI;&@95 zIR9JAzfNxyeAD!=5}q3aC4d__G8ljBUsxeKt{25${4TF}`Py1{(~4|Ykcdw2gTFZR z6;&_h_?v&bt^gr+F7*}&L6@ATP|c|gj?hF8>WHBr6(LlW49N>I#YDg?EJYc`AR$!+ zG`7#eC?0oV2Kx8KMDeV-4n0$>A_%njdqRmI%%J#+x!y7w0d<0ff>C5*;t6C=S1j@DOl=>9L}&au0)hFjA~Dt1z_ZQHhO z+jhma?PSNc&5CW?M#brRPLCevZM;3s$20c&1N+OG>zeaMgp}(LGVxEQib%2;tu`$i zbE{|*D|cu44y=OU)p#^5bx*>gke%Ydd65KhS?N112?-;#tMgWKPAVgbH-h#FqRfj+ zqn%F@=zjWmo?93S7t7z$APqoQg0I$SMSI@GR&smIpzy;K*GvJb;Oc`0ksoqhVS)*V zFjyi#QG)cD!EpvF<`{-z>2?ti2)yq!UfC5?;SVkRPDZA*n|l%UO#ICK^BgTrl*nLl zvj+C4r`F?28kN6c50b%RnAQX3@^%8B%ldP9es_X`ZNiYPpP&VbH-~xDJZ&z9qXN(v-y$UQg#d$Y*wP}nw_cR3Y8h$*nZH&-n;*Y+ENeUopdCYo-Gqv zs4br4B1pB#yeGB6*tn*<(xF_zXb8UoNyOdv{z>o0bJTZYM*QRj|JTDsE#K!`r5^+9 z_p$zW`^(45GQ4%D72TNKnMpJmrBkJ2z3$kATW}$9YJ5(Ptf<}irQO=yQ%eJpLS`7X zJOShQX;k8|Mgxe#aE>b5qP$N4tJAcT#+ea(U!|?Db$K z(;*~PTfo`jbe67-@MXARK6FLE0Tom2K%(g$?R$-*qbvNR&mp(_{wOJ0{omE^1-a?5 zY-6ljPVP~lB=2ygb3VEAw<${)mIzAiG~j~W%SW%Ju2a0-FR+{^S)3dZhmbp4WP#}* z2Bzyu6+O>?>_B7E<>Vf{$7V^-oVi+W#r?9le6zQbEu*F5W|oW@8a;|kgAV>)GOdH_ zw{%{ePiLx%FoGc`65aC|k=$pzIA(~|7+>RohNPFKPqBNJs;Z?6m=Qlc&uSifmtI?N z&@V~~0$K{E`C(C49w^L9GoMaXIRwYQPEZ3Fv4mHy$wS?Frq9F(zbPzXc9kZ34YJaV z-rM=f${0~RD6Kf1PLYMS;lZqy5Ae8hZMT>tb{Ci4xoy_Vb1gT0r)C{v9~gI~S}*Cx zZgXGXR8YNySEM$OqY&xzPf*L^4E|#7xEmN?W)9z+yL2y|MOyYowBIXQfUN@4_3o>L z*i@MEAC7+>IY3|*$zk}kJO##<@T@k5!pCUDS6yLjq2R!J!9KC|oPG{9Q4}Fp#uNr= zj9?+inL1?KJyp!r_E>qV=xf&Oo)7EtLWk(MlI!=IX#T)gRX1vqoIIkDYR{eskI`3G z{3Smjx~5W>N!FY9#s*pjO6udklK`!|z^#>fADUR59bDfPoa^O`Da<>)x-lzht-}`a zZrnxdg?e8CT3c5i5biAdcHxth zf70T=V%9%7eE%O>{I|sQ-+&hXHH6O%9SG>>{~^P5HvQifxc?^OxmiQnd4mnH=R%!e z0TW@VL1umPRD{_wiH)=EZZ(1)4*1l9K?W~ev&FHJR32ydl{Ew)=D zcbHdiffEI>_aatYr8p_&<+$fE^P!g1cdz9$c9h>Lw&x1nes;5ybR_&MrpgL(@(Zb9 zhwAZF_7)>Eph$$TFGmx9%X4)X9zx9sIVJ7SB>JJ!hp;q0n<>v(T5X5KE0SjKx2i-W zC^@c>)QpK}&scrj-y|y(yMXJ#W*cQYZU2T}3-`1KpIstgb(LOxd3iQlys=*PUNOu0 zC&x;FIG}7w%*H4!Or#khQd}B22{y9N*V!F~Cv1Tp|E4s`tow*@+_Gwhc6=t~BCdRo zB&Yd)kouUMW3rC#N`cFP~Ttf5)=4r#*`a=_e2Ul`j2$w~^&-rJKLPbadeDl37>1~xsG>pTgJ?ESBQ1ZM z%g&ZZSY~6uOzh`F7UFlofs76}peT0|vJS?OTN3shBiet~)0Lk`f=kCac}I_zopsvY zVo&PIxn4uk?&{oD3Sc8C9+iVkLj!X!9SIJb@-2qETkpgAaJc|gUqp^dq|k!1WKUus zapwRTBGfDy+LH(9V^%srj}7u_-<538tb`I1RwAqnqo}ejh}F`opIvkJ{6RLYd@I~ z(XU`73qgrSC)eB|Z++Ao z?59q4UY*n{>X&->xdE3+PUiAsw#g+P_1vUT$V_#?8AkhZ1q58*T)R;*?Mg9z8YkA# z`|gKlCy|99j=3$F4HT%usS*VO`0Y0F)0f288nI7I*+K=;)v3lqJg!yzkz(Z6eiIBz zGg%wjbygwHJk!;6C2af5d4ma$rDaMg%b0R*Ak2oq%%Hf}0G?vVjfH+*eo2jJCpGRV zq}3dY7w9^89OT!QV0Y2jjnCF|@dCNO3~0FN0>tIJdZuwPptcUFkhGd(O23Q-Xbt`s z$qT<_lxrt7xAD?Npj@&2A`VfPU6?fmXRHJUW>gG}jO@Z!4>3loi+(=0l3Rc+= zuU)7ocNctkx`am-Q(K%Se~rq8#WiZ-N*$NvPZ_WwFCyNW2ljI8IWn#lyv|Z#zEGIu z@oC7~N;f(aDD?`)qMyzqfn!G#Ekqi8AgiJy6{HD$tXXfskD4adH}drYB?-m>*8WN6 zBAN#}q_)T$-EmM5Rn2#& z@BPAr`z7OFvvpON_6P+&^^C2!Rk|OVLI8AqE#ym0U0|Z{y}7N$!TA72K<^R{qs9nL zbUO<-^b;PA?}_jzX=kCkWO=VRsnVG<3hE7e%_u~|O4W}jnT{I6D`htJ*wmBBxCG-5 zwrj^{!Vp4beX3HRZ)-V8V3+6cvUU7-z}fk%iq`RC#uJ=u1==^gCxi)!3WUsFP2+ zC}n;LWaAeDz$CyR5aaG)|4Au-5@2zNu9+y?uo!orzpLREND>9}-F4CpdVtL$17zyP zI?aRd?N_rZKS=GVBjy}xP`-Q`VwEYf5`n^)4q=#m?XgXO2`OGtusvwFVLw}0iX>*NmI0YF229J9Y~1qgz<4T9NXItt;4(2ERI1Pzj2 zu*w3yrlMIBMaJRQFuIKnWV(&oBEUz*o5lGibb5Wa1Oi!WjS;a-2Ud#H%}jMnt)W)J zpO3yGfeAqMp9S(U$R<3vp5L2S;>jQbrpH}SMymrVuc=5T67-4A1m^wE52m5yQ%B$3hqUevnmEnAoyv70BB*>pnF3^ZQQviIHGK)(Wu%#q5_CnZ z^Q)mAS+3Xb3bXWt>X1^TupRWY(}E?~+Go&IJydCq8f-NHbZ1)Z9d9F^{(1JonV|$# zW$-wKmJ^h2({S`9Sre1wnJ|y|$xq+ZlXXt4-BhVf=cG35<7wM2S^8o1R%BEA+Z9t* zpccd*LJ89}OxO*piU)TOVtkk7-BJ8Py+%KGvpaHZRlCzg^SslF3|sNv>q8qr=kl8ex65oE#>_ z7EH8%a`Z;6UTl8ItFKfIhCQb!8VKxhawE#KmS&qeLBgtY6G#?8^w4)gUQunj)`#t% zywNJ$me~rO5>uoZ@l$;8VEu?R=XQWp;?k_OJY@VqN$SVN!2`21>d7{DP-7L4%3u>- z_A7pMU1y63iYBwoA~o6c4~}G;>o^bT?MM7cS?G=!;j8h=hhr);x!amv+^{9?=G#jx zf<>QsVN<4v2FWC=^*_!DP)VfeR&id9#J!KnlkKE&PZ z-#`Zi1m{x$Jl;6?AUD;xU^J49wuxB!tK7G_qH%l62&yoCYmL8fcZ0s^^gA5Jf^L~B z2Erk>c5X?4K2UTO%9#41-u5R88@N(_JlqAmfH*UDp-`lNHQtR8msKm%twyPE%0=Ru zdm^KX*8QaY)6q_Z=?2>Zv2I_)v>}ooCp^Y|@5#BLRy$Nurl^=x(uRbQ3Q4-DgSqPU z^Ijzm7;yNty5K;j$)*ZQseuk{15{8df*jn=DzqWjq93LGcE_5UCircbm*(d&@}33g>`)L z`=2|&G_)CNv{_7{PE*imm^=wtS)X3{%D}!?EKod+lR7_Yh!;$I*}=nStVya{QEkip5N#BQEvOrF*g@lj?Q93GHzb88xd4+^C$kLF6CBlr zA)3MAIstLALYp`Cn8~bI+N|KmRMCgAr*jT%4Om{ZT?>y-?b+Cm&y<51hkR(Jh~CwQ z8IY7V71aq=G%)_(#-2U4ymRxJ0lB%APs1q-W3K{+r83&rjw!Vd=~$o(I1C3We)jiS zZpEUmg{f%60g0zQhPDS`nklL9Ga)=@3L0r-X3IhfDy@|q6IfR6$xO5e;qm(Hl@Hoh zzJny|BuW6J8^naBoCMiHy{#bGS15`u$4gwD+2KKVeqP^$9KbL3*z3=Omrr{<-w$!O zkK-78yB&Y~+oPA~tVVV{zn9~wgFii-z84qx+wZ<^-xF#yB&cc!j5|{LQ3J|Zl}oRz zAw{(1(Q2P8@B7tj*gJSTeqqZNHxNk{umxD$LFZDRMwvs!_{m+G<&0&<7-^pjS=aB2 zd{X=ILx%F~wC9Bi6ceTh_#dw?wXJbI?;g)@;&DCSBBUjvENh^2WL896Odv<_aBMG< zcKsRg=a;}^lE@*&Q+|DI;~sedZN6=u*3ZK`ao4@w&|1H!q5U>vg>dRL&mpiJQA*aa z;i4E6+c&+m(1^p4WY4?XHlGF!u!0531}!bQ4QrQp6^l1$*FuzrUAmd}d7)$HwpB0A$#V73Q>99K3)<-h@ze3wDS|C9dn*GvAh| z+Ky-xd%?j`cYDE00$yeBmm7D#LX7*<33bNkO^^H`IA0_EFjCUT=IKLRM}(**hR zrdZeen1y`hCP0W3*qh|Ctvy*OLhqM>+e6PdJ;%;vmNu(CG}HqIvvYV+>olH~5a-}D zt1a)*XQB&Nel*oXV9WuFghsdOA%&+q{loz-UvTXkS9e`|I7^{+*pgaB|8w#~^EKXO z#Zsb2ih}G189YTB8jCI(B}oNf zGSL}%-fk{8d?Xob*6y=@d@pWp2LK9^Ft5kgoHc9xgeY9zHfQhmk;|M`O24GEsq5bN z)9W0-x3|~Fp00=A$67CNSOQ^9o}qih?wtwk)KmhxSr%D1EfE-Wrk0;#MO1Lmz1f z6l9DlpZxrOE`Rzjm)~!3!BJFVX4qdcReV0arT`E-=ZqVOo#V5Y&+4G=E;sshd{Z`b zHqFHqI_p|A24!%ok!>1_N*H1Ak!ES4&&)8 zQYId^`Pje-Ict`=q4nf2;~!sTl|#?Bj1yJnAdFg*&_0}B1taOte2&!;4XI*bag1wq zkBip(UOHhqp%+V~ZO3(sR+`&2SI+V~dB|gS)1-{s6xy-HEu$^gy7rWn$QB)|KFdQ$ zBPum&6l^#WUp98-qk?d(lRosao*Noy)mv8ZdYSDfa-oivWvgbm>``FkX`vB=P0jH< z?G}3DJy#Z3>owVGs*S$rq3a3h){Qf$e(;!$h~S-*iV%$nCv4I_Cy8=AIFJ#g2{^MIVG4DbJMgbmT&7-y8u~49RElK#sA$u2}xICg} z!Y|{|rl)L<3>>NMSeKjaqTyGNA?-c*VmXTCabVdnF$TjI46E(-_J%UjDzY^! z7lrK45SiR+#T2aHqF?BK`X{VwC#Qk@1*MYx4eYA2HLb=R#|UxklxkXz^O`c!0b_Aw~v zE=6d?ssrc&9=awoTL%W z73l*{boNHyXdfKzA8yytn+EP*o5uByFycPpC!W7A{RpY7xXwL0;YpPqlVg!Ext4Kv zbyfY3=MM=~_tO>&&S*EowM%Xof9-zWg!9iAWQ10(oC!g1?JfzSm({aPUi<-lMTYP% zJ{!-K;k;j63hNgxZ-Y$>ocN^LK?dqxxG@FK%?zi_UA-{?m2#(6ZamllJFFOS$x>%V z2{w8NZCZrNE{2FC$4QvdIYfuH^&4_SFFuJu?At8Ah#)(b^;4H?(7W*NOc^4ePvnmY z-2xc*EsZ&TBIWeHR;JSSGyVLt&F?&havFX9di_GlUrO+S8narDTgWa1ay{l{Bbw6&S#Hx$chTkMiw?SR+-4#{R@j63 z4ZKfGpB;-+P;bC2`y&6E@tSz@KVm9RYTdeWfYmx#utWfepuE4SVE394Vhq#;wT_Pq z_m=xYCHxL!2DoEN+`Yj<&+=#Z_W++m@AIe71Yxk>2Dsd~RN{@El9 z(A6Ex7axJ?llhO^L3uZDh?}mj-K3qH?>7Ly-&;^v-sin|&dXQ7{P#t_c`t{@m2lwm-^PfSSTTtY!tfi~_S)1DtcxVyQEi}vZc!D3!6%!b z?Iwlw)4bzIk*Km8!RluM9N?pAFLM^AVMDG6alGj75v&EUK@$LDFY|4kE~AO~!6J-% zA&Km9)|6=kpB6Ja`i3or^yswZ56=oQLH2en?BH^C*dE?k!4J}yZz9+~f^BT)qWxtH zgdiZCSTNyh8&wh!Rhr%+m>Vm< z0*P46#ASiR-wZ!Xs@?rDE^xle<^?aI6!il_>}_+>vvrJccdrP(#wi44$ElKH`eos^ zsV#+!j2xUSH5FkrexeA07cb}k_S%2?@v=p&z&{#KqPQP6SnYx?qGSufuVlYak00yR z{<}3{&nfK>iW1+)j6UwiWmtIEJX((fyIA(m2eSLLVC_i>+LHB%t&BG2*!X;7Orjbh z2}~V$s7hqC3=|WE>iJ-VC-waPi-WutBM=`7@FY%+0>zFq_aNF?>RIDcQFXBnqTrlJW7LLbT+sh`(;uv9t z{lQ@qBsumqEVUXBre?&t&b28zmVz!Bpx5)m=Ie`|FeS}u+e=YExg7q&|AMH}WBcWM z2bg*0|B`K-yaY-TUU7iePQ}<3()a(wKP}F}b2|!bsGiF8lyAc~jkIEsB4t!Huj%4y zyj5ALz84vp(lp!=1=!w>WixBC9W3{232`akPZB?cGG{p_&@ zZU!z%*ZM&)q6Xvs6C4>)vVrpuiR{mM067DdxHE7+rn03attT>CiVDov?p7T9_ABn9 zk#fp_Wz%OnDzV2E?#V|h^y|Z1DB0eCXUET-Lnr4=?j^g%-hRoMOoN^TK2gn&iV#>7 z&sQ}FY5ams*NKr+_9Djd-YuOW0&)qzwrCSzh9@qHzHEZ3-Ly?q*J#PuB0%uI=h#DE zjt0BgV|d7*g8;jfh5h3l={`wfT5y`HV2#Fj?6w#f&SX+G7U5sZS_QlX zSLY1ah8J!=>d)JRj-NwwK}IrW6@hZ?GpH}GU;=I&Tu*?iSWHTZD1SKU{?wp;!twzyM^Y~OD0_~+%o zYv(+mp-tZR4H%zvl_sRzpFo|4RyoI%g!VD47B15Yhw5}jQx$zWaIm1YtkwDNIg`V; zd@%Z>A53iVDt9=*J!%-|^8B2WJef+Fcu&%h>A97qQgW`TrMAl zC42!>`K6_z5+W}gf47b?QO9(u0!nJW7 zNu5R&cvliwSP8B0#(4879ci6oYmcFM8!C?yV2PEx%P`ZA@ZvLb|JAS?!;=f&X(15# z3)X#q?)cm-sN;VfHwU$Tn|}Yg>EC?VmACi(BG<_=vPXG|bG$a7ES#qKg|T*BV*BUYN-`W>Awh9Kl44-SnkzaIUe*+8%k`y0$O6 zs=`lXEL~9+tI9^E%>q3(Z-4Yd68r7bhSx04X@Mdy!W<#doa~6raRuwM*YH@}z$1ue zgyzpJ24!<_VF zC+0Y7|6m&jCRjzb_n_TO2``uTQ!dA@2!|uoS-kX7bUCF8!%b3lV1ep)!8*;!wOH3L z;Kd-!iiD8mNUZj}B1Hk{)7}Y97m7Tppneh-g_&YbGC?#DS>SH8xD6x;AHMw$u%}pm zPHt0Pg4Fp2qM8>LSglmS)B0>Qcu#q((Pqw%0ifeBC9y1=RNlAP%bAl++$8;-yBfPa zp;gV`K^-4XIOgE(u&?<0oRq?oGZ^XMR4N9jlWJuv$}zZOv_C%sQWA14_YrW+uTAmV zM3{x){@^C5!nroDl3zhEv}>B+vY-57cV}=WKFx+JT3tlOoDGO;5A*!;p37XS!lID72*=TBKZ}i^v7gZsX zZmYEy2(_MSWv`aI&n3>7gmfa(b1hB!X^`n$wGQ9wuE2!bFysv5aoLDr2WBkRADbSe z-z}6;v2EE~YJhVYQ0Dp8Af(weEHEd1@%mLe^ZuI5kzLeTX;bjr3ETT69LuDR_AS~u zbSP><=Fj+Ub&^%^xlkFzj6oqa_X*>Ds#h8`!9?ohNaf>s?ERw1mvI?Lg;`Zw5-r=Z zaVeoCs4$zb@JOYTe6!p-6Y%(Mh`cV*r2C-v8~%^mq`zN-NTwbP#}DrOTpky^6r#l| zQ=z%7_3PqC0W5=iD1eKN-cM#?BE z0*?<$PKh9TA1j7T9#FK-aIAh6xzS~PXdJ7k`qW=%TguS%-*t2TJV;9u0HEbZf8~TP zX~5im(!~ZZX>**=U%R-zCgUd|uWDhELYKP5au%N`;q{p$gZfZZlP_Pd1so8I{9J?w z`t1`n&}Cfh`1cA!dZlWIV(j6_0b7_*-`g*E#1t-~!nRt#o_TyE`!`e}E@7au;O?m^ zfJmh16)OyX-R^cmaF93Z_T2v1u!0V4yO(G!IcJT&G#Cx|;~uii=w?bnt`$Zk8B0|N zUU>@6tP)@#p;qO?m^BO)$9n82Y|ONVHELv5BpLt33nmYy~318D>u-jIZBiE zc8ENCl#Gdr4?bqg2cWBnLTT{Z)hhmdMf^%T@OWKd7mp142X?i>n!)yGX{s$sb+S|U z_g5SO--|vaD+IoDVVRMPQ3C}$C~Z!vl+%wANVH5oOdPc?A+1_osI=kEdB$~Wqm!bZ zr+mR=Edf1+(8!u9=EbpiMPW13Rpt$Gd^uE6afdPV165nfEV65iEOnhEVLoCFUG4fn zm2{oxxJ-kJD`OH^8|2d2Hxm_+O(~Elqp(BDx6@%(gXj(u^%d7kFF+T&!viRwjHK^SPp5 z);OV7UzN63`8;joKQv)rjsmLocacA4; zXRwgQN}LkIvvCSZLh4LZXpe}6F&cT`(bKwylz`{<8-P#{GF5N~kV?CQg%G`x5%K59 zpLY@)H0h`8L&CfeePDOw7_na3CDK<&LUC()z*OyTm zc4r)FY4?eeO^ncef8ZJc)mq>mft!I}1gjFOiZc7Et54!1Sp}bMWa=}s+X~NUC^NUg zVCjeGkXh5xlIgn{iYI{dk7DLFPeVt#lX>Iy59}ewI_aA0M$uymRAt>cbKG|>A4Jw z*;12iKN?eEaGEEiIaS*UmPg`5heLTHIOM2`E{Db_D04Q*b6rxisSj82s6 zdOj0^(BBA1^4#jCDbgs|NC%P`;E6B~jL~fh(8@hcd*kc|4Z@Z=7yBSbxbwt8rAddp zuI4{OmY(*trlbnaJ*=z=#STe;5tPst4Qpl0rcemst^-l>R4cqy_6$>VX9cE4a`}tt5BV|f z1g5Ad_Ltvo`DbG-fAh2_IjR&int`cRcF_z&5+@a^)EqRl9OUI2)Vf8?0~@A5o7pD z1X$ADyfqK^N!_y3Jc`HqiI7hi(MJ|~>{DLL`byb`Odlqs@_ycm;ULl`K9>-Rg%t)^ z50df09l-IqA)D>_B(zQz&s7ME=kR?H zR{QPt9hArOa)50RO4=`WY7e%x(f0lrqmJ)=za8{mJFpVg>v`30{{3}3xU!0TiQfC- z@qN&G1SF1^^>aj2{cx#V` z0p}6#p*G>qekh)k>>||%XoOjYIJmwV0%EOzu%ZTvcGK-UWz7LU52XSI?7I}E0#jl= z4dfs%9^#3O*~Xn?#P2FoT}f2{QN{qCptOuIX*S>^WB-7Bm)|2B@X8PKDrRG3^bdqL z@n^!W2<~&}tx8f`0l|dDxQC#s8~QH-B8G&WA;XI`Ys&ILqnoh@?4}6}I*U0D5o?Sc zQ9)kdf8LIA5O}i;0iVnp^``_q+*;H>=|Tg{4Aq5q_^xMT6mJ=zcIK@-WlrueBc3$pgik^bIye{z zad*$Kp<_ITKOkdq;;#-52FcRX3R#nfkY)Jwo^#u&XB_0aE>zkj@2eowR6iM5vCX{FpR>$gqgL;&y)+04PGsK&C& zxo78KA@E5(7nKO!l`>E!RlpC?{_)s%UeB_hn*&5q_nAYRNQ;d`%TtmW9h)+@KA)Fky+ul5xg5MU7kVPwaj_ zl7)3EQX;;Lm0M4yTFV*z`!2IzpAx04x}&hrX26KHkr`huqW~5xSsYRt2-Waw!gEhzv`%sZMgwi1wM)v3 zI16QiC+)3q(M+%hCvNlZ!USc+9p~k}N5jw<-u$q*pm8>{M4*SfrGX&TK>0>kcHNPs zng#*U?j~9EbYXLc@fnu2Y;eNAc53Lkarol0#Jh5!sv&e9O6S(eaW5nNqvs{lbYcW+ zUEpZqmg@YFQ3Fj!OD_|*lsltD1z;#xPcZ)=zs&6SSsca{b)Cd+&d#RiUd9l%Q4iGicB@in`GEd z!W8owXW^4rAkHy-e9oO@(1ejUOFuJDw>#7(!SQQA8wJXygL4fyU5>g)8 zm{Hw>QrfWB+t#co1vinn&71oaD|xg1#s6*j9cIth_Pspv(*LQB-}8l}nzQ9;M_vR+ zx*lNnxaC%ug_Du~rPDerYS}t}z~dg1b0IY5BAUPGoR{w~)-TWcJi1X=u-S6{qVRXc z5W&&}{c>y4c#d%g{S-IKbVEzQktf%G>qPrVhY!NuN4Y)hu%xl>dMI`&MLSf315&TZ zJERe+g?p$GbA~jzuz7AcuC#V;Fm5FiM|m{3B%J89eZDJ!Dy7hbvt~$Ac;sGw(7lb! z(;>y%sUTTy)GgDUEFy1jOx^Py>ns>U<66zd+d%b{Vb(oWl)3$TezAf$Q#B*e)#*z^ zbeU2$Hr>^8M3XnMj%!^KN((;py`y6vKw?^CYv|H(7)S1TDvR|loHn7=!Vnwa#3*fu zaZ?r}scO-lbM&Fk+ooiOBVArL%At0`Fd!pc=_-<;>H!+q#RVs*CL2jnNK;FPUn~M) z^tnFwDD&61ZLPe5@pf?WZl8X zOxOdY7V(y`pc?cMxzwwx4p?kLC}n}a$PBV6Z|Bmg@<#1K*@pxgmo|-AHt0iXg?g}R zA)c6Y;!Mzyoe&4(I2%n1wpNH=sk0QR&vZzRMBMl-_Sc{GiDUr^3h8imYP~s9A0!-4 zsN|>AdzHP2+l;_&H=HQL3vUcZ9>xgz$3jzv_KWo4g~2S(QAN+U#WVy!+lYO5V-u<} zI)P9<23>qdlAV;9q7@(8#;JqtqFPv-kO?9CemsWfI}9j!C7pc@RiV`=-65Hd3;z8q zpaq|qD);OV)^vFy`SW5`5KHT*Kr23{-H^T_XPpSYZx3>$4E2Q*+l)hGP?ZZ6a1hN< zH<*BS>r&($QMZMQ0B*R_mS20@X7i~)jKeu(K&zBJhSsaHZjaVJx86LD6{DOxG#blB zpgiYT#{qn$!5*G*5rWK|oc(SIkJyS2amw*y%TH~PArCbNbyvceAr{( z%#y6>eiaZjrh#g?OPgg71JED$w=(90&^+UzNuyy3Fff&gn4KAGfKjE({m6E$^hU!U z4*ug8)_rl%@4>N*Ffcx&V|4o()l$@Z#VY0nCz`Nv@f%vF`f`4EJwnooG>S@nMs&V| zq})>!)a}}nd%pqFZ8lT6HZ3n@@k}S0jqL`sGyS(gtPtUSD*eZA;(4U@IiyFgdGkiJ zu@10IFAqi8{54F`l7aEKM@Mht(fd{U(wsRc>-9T}d&BQW4w9Hh>?~%UGpn#O1W6@> zspGPu2WW|hB9d1QVhEq}Ekxp5WpbnoIP^rLEy#tD5Nm7euL_XGch((^d<_lv|me}~cC4Y+ljk*Fq%OlTO%d5HYmPY^8L)3cZ%h|~HJ&(?2-c_Wz zH?m0{6^4qAkn&WN_)<*i|7x{W4-p(V|-$->u3XR~xIel!_@ujWq5m$D9elY|{(^R;{2r;A9E{H;t1yn?%T&F*@^0i} zFIqRz`hOgupErEsTxiBBF;O9snhp!#$zZCy0Ft^|8Fvm1Ag$5lCvUOF0#g<|syb@- zQw?#l2n)9IZUqT92OyYt)48Z7srDWF`W%YI+JKiZ(!}a(kL03UYr_E1kAe}N8he}* zvegf!=kARi3;&D2R>+{SqrE(TI@Zt(UKwVRiTSKCv};1Y%-*+k*(xV#3cQcKpeXSR zM7~VxNa74p?ORX>N72l)pYu~{!VXn;a+M=vsjVb_DKn(+p;NT^eK7VwF{;Y~B(-cN zq3js%tFdNV;$lOx?y+JA++QSDY?*JE-=Buur%w%{MZPA^B8!JTA8G+#bIDjDG5%5cu!2^M!mg0R_ECy#S^PdyGS3^$~mvr9H@Px z=h1-!(o%6%*#k;k0FR)fu0k@B$uykkKzZ6YzgEBLBsvVSeezm`gk~GcINDV-hSmL~ zNOArCk4Q$X>$SAZt;CsWhjINjO+5$Mcv&o)Rt+?YhgXy)#>eOxxI%CWyNS${c<5rC zjKEe}OiprOVjQw7R|esFnYLzZAPPjeC85%eb53Wh7yP0p4sO$#(E)W1(-Uq z%n@I^J{6VnaKqyey$uxXXXj?skv|I>YX_wMAjO9TY;k5=a2OM(7B?$duO zsQGUu1|rJ@0z&^k+&OAqfR$(kw|d2mji(2LypjT6w1}q6k`%D zHyTP$#PI*n0u63uJWbL_ithgc0hRDSq!sZ05b)3wkH2q&(~Hh4SMK zytf3(*Jo}?Hi3-4%j@obE$Z@>oxXQ6xA!e7ZO8Zf2!Xz_R?p}2WT&f1TA%M*kFQsT z^QCmS>ooEvRaaA;mWA^E>|v>kI_+UUY+wA3U?+;a#wV3Niu?oJs1Q_##l727pGXgq z{-Sc`Z*5MPG6VzDD74a~qX{KhJbwLZvQp7Mwyc%Pue$EB(k|Dr6F)qf5^z7ZuMwST=`H3rGEm;383#h z5e&)W1X7sosByrug6mX%w7uNF+l$&SXHQgIvJ`?^mANdXJh<`Xve>8U!f9}hZ|pzJ zG*~j0m)70hn=3}-kq?VzHj%%U`n_=q^llPcWma?fLkSF}-M##A)iizeri+yCu;ax0 zP6dDE(T@zf$`kmce10jkj`UVk8`Yn>AXZ#Wf$_iJvROvMrSZH&_MJ$4YYJw2`CdS+ znhdd^9qX{ZhXM-Jl!*c?IZ{aw%1%F&BpT~hOYH`$-_kd|p0dK0^H77!i$2w!xT?Y; z+|a6v`!}1ho+Y9)fDr7mAsq&iO3YqyyjB|BYtYOYPftqpP%Z++(l2XjgXeywQU)IgfQ!BV8~`{o!hQGOL>K*;bLYWK_3- z_OQm_=|Hef9*X2YpEO({UD9V}WH9C%*epJ6k_fTSm^YXUt~^Zx`R=Q`OdbKL{C@Cx zm_Qd4@u~`HxzOwRUXr=XgPjPEsfLwKW|ZTm|Eto)qOIPRYz>K#YB89#RoZHwr_}`9 z2q@J+?WYSV3dXGEhN!=hB*rOu9*K8Km22rR7DHg;->OX9QZb|Jpddp7L(2FV`KnzLRkp(>iT5jJ~?KL z*hs5p3Wp!qu((`813{U9hAFCDlUf{pj+i;8rCeCwu*seiRm1~SRlUV1quZEO`HFq- ze0K4A`LCMmTwqDLutA_~NnKadW~$3X=?nCINO9QJf`Aj-WfbQB!Q5BIMz(C-wQ0=E z%*@QpIA$0#Gcz+YGcz-L%*^buJ!WQR%G^9f@8%^p`u^o>sa5JOol>-P7h;%r*wlxX+6yg}d{&Xl1jc6sg@vZ&sXm;X* z5>k3Efox=BseW5hy}E^m4CY%=lzzxSoR&ZmOH(jWczLub!3YD+!IArz2(n^IS&UIJ z*4mx!{>Hw@(yhYX#tDFs{0-b+=p6%!__l8x=(ZQ0ycvR3$K$)-=eHrT4RL`E-UW)t zSqysOBFpZb^B?UqEy9!;Co|%;`naceeJGC~Fv;M*)Na~SRuetyDLSL(#Mo%DiM!1+ z4H6uY&GFDW^!+P6_Gct#UkeuFi$p3Wy0`@&E3xxZ-vYB+Q**n1{+2`*oq_U_NU|Xe z*(gy6k{HAlI*y1+h`q2MsobrNn&^16-f>^RX~C4T=qf~T{lY+e_PfUZ#M1F^c{DUaW#63>P# zxu$aKqk|Os%h|e*XbfyfR5|x$J2_1khj`sCz>9{4Vo)3ZZp|W~8c$%=iY)4F2_4)u z>&f3fM?-H&Gh)kbBk9GlRks_SO;hFI&tfXrnBBU=QB~&NC>@4R2dES!y-;^r8n#Kb z^g#*dy7jAnJ0G9X1@GZn-A}BpAyvAFG!hLIZ8;lA;b;p_3DP0PmBDPScGPxYq=UBO z2Gs0Oql2V(z#k2g8cI9_`kv#A1}>LpHC2&P{;DW}3g=$N*}OP`*i9!<==Wp2yp=s+ zC%L*3%qXx9L}JWXQ`w)RsvGCf7Q9q+DGb4V(~UTz<{3_`w{seE$A^&4v0(1Erw5JT zy4#qb0&9?Yn4Y4RqB+6@XmD{0lLBHF4mvJN+Y2l&Q^)KBb458b^+!UWk+yYT%?yI=EYl`WZvqKvuGBN8sq z&UmTK$Ld**vpSQ9 z#WRJLXWqr_nVOYvAR-J5Tf|LSQ8h$+KB*4la#zu2_u-&uKXyvii0j(TJXRdAt`ap6 z1;H&1t&KoVgqeMML#hgFf7Tt+xck>Mi z96qbqdq}v~KnqFiVjWDRa_I9;VQ*Hp?n8-ur741!(8U5I4Pq_!Cj;0)r&>L;AiTt1 z+9E+7!H9i=?3!-wcHDN^c;&1lAVN=n-wN)U+@HhkyG)~6Gj_%y8j@J9`_$?xy>mS7 zO+!Q-5X{y9v6Ebc`~>N7qI_Z=`mUu`1%xSJ{mj0r3gPyKnGdA@`ApajbSZ?`wkx6U ztH<4*MeTNR@eGNj{G9zTyCTz&VMS4_z`#}keMg*t8D7HhJ?AJ{%UgwZG_?Youq3ZE z)RPZe4~!)+h{d6_U!-+Gbv~b@e-z>xWF82!@H@vg`DE+#j~jpD#4!cpcL22K)P7p3 zRF1JMjI-7CQ~gjF8X~y@O-h}>!2Sleijb;Zk@wdmQDY+R6_T6Q*8XR=y5n&Y9uz4! zJ~8B|ZiW0azDoYP^TSSXj#NZ{*#sct;F(k<8ts8)VD|UEK-BlI5THt~GAt?ngsP#O z-wYP8%}U~e_Ltu`fWF-&O1Mor`}eA+=Rq`MklWJlK)zTFB1^YJL`m<+AZ>OtTzH-m zo(Qqx4OR23s(x@ zMRLcZx9<2t1$*v=(IYml8aCjO$m^Xn;GQ;Q!4K4pv9-vkuZ~X8a4anz zd-r1gfRPPgt+}g+cW!<9pq~lgu1H^j@y~N z${TsSw5cE+9!Hx&2$oWAa1rcyFN^e5Y^p}Ts68_IXT-&}p3^nBg%e?Dz}UN7K(`set=)F%bf)Rq)6H*@vUa(fWlz}mWF zSPO@P{+O1cm_@`H4mea(g}n!1A11b|A}VE`N)q9wCVSG|i?}3}>BV_1kZf%duMq%^ zX?V<%{3htQ1f@6BRO6RMOC85dyGVJ=^&p&y`>oX6LCimAnt(0(*-AGT?@>y+(s$?B zvC@hJVlHrefWoGgIz~B-kV|kTOJHXG3_tbKOy|3tAY?Z}KrZ~S4K`uh%D3LY{W(Gb z`|VmX3U5r)=P@YFo`53t@=gd z%3%HCIl_V%-q5C#3~+39klh`_lxH7ME5!TJ%9aATaCwO>29^blS2 z48NM&f0Y{phKRCMu?eVV)mVoICF4VCc6NtEt#Flpv&#heAxsId!-&Jpp}$aj5rN2P z2TogztFp?C&+toQ;#KTj4!TSDUmr(_eLK>qvJxFwW*$_E$ZsjM=VdLWr&syyi6YP) zT*7f1UPM|R0S4h&l>8If!U1Pt2}x8sF*_;=4Eo7 z_`or|XNq1&v0JUk=cL2ywtyueZ(yO@zqU6~ICq5vVF}?t?3Z<*+Mr1DB&~n7I;{KXR)6639-*MU7T>vv;lG!3huj)cDyZ zSOS(+6B`Rm{uW{0+{S!?ZjWR%`l zHAS)*1J8@L@wNToym_`88=7+Btcfl*u#gqM>&0<9qvB5^G#6uGE4E*^32|BTiEBam zeix?b{jn6pqV$U7y`teihfY;QhItcic+A*7DXMqF;I*iCX;Cn?yV4F5rIFy*aqGwb zYPbO0pvX5I+lMuLWZx~XsKD6qPHFam^rM8(&8oVpn*88*9F~cbK4^dwlT)t5+)=(E z2|X2bHKWS0WFQS5w#a$McH&PAD%XhcF4?HIyy?<#ktsO~!+ZzH8Kh$TD@_NXa;q-B z+z8U~fw_&Qf;#;Rk?&G;jIGq=pBgLsY0oJi97(H+&MjI<_;R<(*6fqg*`wmw(^QG` zQr8nImdbsyiux03brp4Yz~6%+Pd)@0j*a7U(QHy5!|Sm8Q%yK(w#B&~qexy6tk~yl21-k*GVraj!-drN)op-3qk1u>WzSU;B;&R!leasI8iJ za#P-x2a1pxaOQxk3i`GcWm*K(wd!6ei-}RqWylPI@=$CS#XVLZXtt8FD_gLs_f}zv zzz}%w38@$-h$XG54nK_oiyJwUG%D|R@*{E@(n!E?3zTWFsBj@wp5)@Z#sQkR8uyQy zBFO8V+QZP^NhZ|`QaP*|46155OL6UEX3;8#J?{et@Qi}v~zuQOA$i61w$ONX@5BXc2_9m?;pUaRZ zv}RQpSlFo0CkNx$c23Szx)m{E9%sX?=*6cAj%?QKK59=DqroZd_;cqLup}Ts=FAQ5 zjbHr`*O+5@?N6(0W7>39i&J*!!?yrpo}8jrRbWWALx!@kDuLrhifewiVsAVTWsj6Zs5YZD|wZQ(A7KZ+C`>0mjj!Ngq18>vT)7bl7iG{cnkB zlOj$i+lBfPxFZI?RHalQwo~B5m*;oM?uZ3u8sewR77z23nxUh(i`Pf%c%-Zmp9C4T zaI>i&WF`|My#3DTLl{LCVAzc@*>E*CxvL7_^#_0fsZbG)|Ts*XW<8pGlJ24>Nl z^}v~yg#2|cBc^mN-Ec2;ce2<~_9cO6H{J=W!0O|HdZUecL7mcPZxP1xMueX9z9S|J zrt==0tF5`zGES5kt0HbuDE2KhpPQ>yh0>`{+#>KbJ>8SQ*N`6f#DhsJQa@E+D*l9S zM5ikeMt59E?bVv+H`E|1VME1{(@wINzKOrpPP?F#80w02IEH`5PU;^gU)^yQB}5LV zv{B?XM7PA#cub-Xb6oF8{RRRQN+rJA7^2WNq^*Uf%DDaU_3Mb>x4B4ian$f?67k@6 zp!VeNy8{Oa~c|_ia7sr?UbN$Z5Qy)P69KOeS zlpj|A{%RS@xc0s-ER_)2q!>-Ac@m8g#HodoFgtsQP0#_R54o55kj3q1cmJ-Pr2!Ij z8^8@MLlRsBWC|hi+iBF66N9cS_@Y!I@wgkpp|mUSXVt~+(0u1>1y^|YyR?eL3rmYj zzt+{ai)c*2PstKKnGdy;V;X|eibQKoVg5%O5s(s{>8i|B?&Bp#%n9C z@=~x7b9y{mto?DjC?y7Igu9*gF1=dTTfbmJ>DoeEQOUR|2|Mc;>s0k89+e1ZK9lyZ zE(ljlS&7oANA3*V6cq7DL(&ebWP}yds^u{%g@^~YY%W{&3Gxl8VYt}9GaC+itBnpP z(?iPme0&Z$r}>cNKg7!6=XN90KNGiN4MUG7*GtK1g~@ zRrAU;DrXJ_=W&c@xM0zr6zw<0flLAvAsf83>5xB=G=QEQv(izw8}gR#?&z<#of*~& z*}e6CnRH>wX*@AoVwxQ1fLJy_De(8=eV$+T2q?%&xatU--uz0@2W(JL+Rt zOAx?cWiNUb5=HQWm)#7-)-zw#Jnd(L%hr?!{t|;%?qe@0_i=$Ub5`i}+2luN%aC)u zUdsKb`Qv#)zW+mODnodiOvvNI1=Uv^%jY#qnea;CD#UcjD6y*ox^jfo*gEPG1q^F= zdP!jE{>P@mujznJ>Dv^A8DoYd68?(JOR2Z)R5`k1`7rQgt+`%TVBEkg2V%N#zOT78 zwMCd!&dgp8JcC7yqexpd$Jy-n`zznCk)986729eglAJK31FQl)7K8D};Vh9>aiOcNGbdyU@a*k4+efP5h3}HFWA>r#-b5#FQS-@O8dpn$HwOJHs$CHc+Qgl&V>WX`Ik{ zxx-zHs!+7J4J}M_6w}IDVd?y>fd%=}lLcWs9It&t+N6!gBiR*7UpHF&S_PBfcmv@S zj|6*GY?YuW|1_R%Q$ktfWP_rsb|DFz>3jtFJd}9*g5_vSHAw&fb}7*Aw-Gq`i&bW1 zSVLw0jNz9``fNDrO`P#HooA{QCj>cp{VAH`?&#%#@pNeDPDYrtyT8&FTr(BZQPdjf zLoyrY*q)myl?;DvhwJPjxGbK;L$BkO$kKhH z`-Df$1bCH%rfs+4GDwT)$pxR8h@7e{aamgWPr((bm)(_$U@aZVS`Od2FeGYe zB&r?QYTc47KT0A_;b}9Y<{Y3O`SbMj`ccwxv6gCE zDkkxa9B}Ku$z-rk)WB_|lQj8u)#i^+Y|-^WSIyZ=)$U>Puf0^29&8PuM%p-^sxe)f zX)D|%sI58(oxPryZ;&L2Z-|s&+^gFVA9f=6@V~6zR)H~BZOA-FHJ%ZHzqRA6w^e=< z3vGVAi&cQ-z6&(zO&diylA_;&tgrX92VHWbkM)dQD$t|) z;JWSTDNaJn6jLbl*oDgK@<5JbN3zi)eb#yoam;M2H)(P)-cNVGAel0HMvQX`d}`#Z>Yk}qNX z-NfN}UkT)21H96)#Vf&bTm`nT2-8Z0aT;IAUnKUdL+{`o-tKUJdsKNr#dr~m-& zgAp|TYX8>@@jr@ae-+T^Rg_@?;52`nKK}Z_f4O})yFmkhKp%hr08o@aSN?la{yyAa zDg)U6JSl&T{J#zS|9Lhb01$w`izICQ5j4d5kJ&JAHnjXxRs2td0qQ?f^4I%6rX^b0 zGIC7-mTScYBgxX&}p~yt3qd#yQHqp^{1rQtBNl43?)M2(HxO zsdJ$+s2j>)PGAm8j~f$ZlrtQ1M_u`Ny_{Bmq_^U|MHajAdxI|O*vQA3Y7r3gVuI}Z zyiN_2O@IlvmhF`&$HhVH+U!;_yH(qCXGO1IjAh2kyWL^k_i5IY++BTJ_V{83Jpi}( zek^8#+e9Gahq#FkuKgrFL=2qZ*>g{_O$~xU8ih=cYX|5^gu$=Ne$ zVp~cq_aY&h%d3RFG)JwH{-fQ8F8f;x+i`H>sz_?l;yP;k%)^Vkg>2MW3vCrpuu8XV za~4h+{t)EDlL*hMHk_jLR7VeC4a(#6B~E1br!&st=O4`zuC@%)od{C-u2=h9_r75o zz31#GJ@md0F|o?m?2+^hyhq@=sqHUZdNWp8DagI=BsWUktTbevzYJD^8j^S6TydFPi#16bnE7e6C>puUdfCn{%Bo+%W z0Pxox`0vpA`@Q*BXp#PNX#GEK)BgrR_IF)SPAC8X^*@5l)xq4!*x`Rw?aQFLO#VSP zdPzmoHj@FxXQuZ0H7;Oq^laZGeN9{EakgTqDb!pjC~X)D$mpC@d)#-6lMP5%CiB8N zW%S@#tJ|@AMocA2*+KNg9~~&(zB&KCX0+kef=Qt+m8u-HQct(e?Exi@ehskRMjZ)D zPcd|4r_FmX=GsRJJsK_bKyQDKY~Oz;E~REwju$OU31{5XXR1VnSZ}o5k2pa7>=hTR zSEo@ys)s@qzu!Tl(aqF;lzJ0@Lk5}>Luf6N7u^#fV~t6=Fo7`;T_1I@#nj^=17(r0&Y~X*zZ2n<4I|+%QSg>AIH*ym!LE^ zm}0Qse^Y7&FajQDcsd(UYZ(k7gAT)-;>mYEXOcD5fw3M&9rX}bkYoybdj0E^pw<_w zBeI>R-e!-C!aX~iK?F)igM|%84iM6ykJC@Fgc*rw*}b&r z6r%S$v=t0{LmV84D>~n*o*cM!3d~C?9=3D~<}_N5rP)jNw9UU;2?%n(0liyP?^paf z_)-CX*oXfMcz|Um-EI6y1b;=zzZ1dV@?)1+&jhf_~LgmE`w zE|G{qC$0Ijg-$*XM3sPk73^0X?7$sw1KrlSo&|UVG zPdD}fOUYowcdvCBq6t|T;TMO-)c#Wz+4->HkUm?OJ@f6?k^^@y&VtbUc72>}a9V4XD*HbhK?!jz+(1KYwRQcAh!>?Om`3D;6 zBZr4She+rS1tv#C!tmu0N-vrjd#KoDmc=BKuni|c+8SiKmgz8ec(A_Rm`Mnz971f} z7lo$rpbWRf+;Y@7M0i`{Y~|oLp4aY~rDw`n_kIa>?`f{s?2-Ni6r7-vr z{t)7Dnf&-wS8F$BaF!X&KZSo~IiU?%vm)5L857mA?77AOtQ&{Ypi>(wreW%3QscM{ zg5>;wb5)yu={q43)^hv7GV|^_Ad=F<8lFzgb2L%t{;7~DcqHIuRsNQNfbQLJ^mvrl`CFc@#QUP=`>GdEM1I$@a zGB$e(NmWlSp%JWQw-h;VitV~*YWMX0kQ*qRyu#{RhhQRvU(PuoWtuuc5H$T_+vF){ z!Y1Iqs50CRijKZ*54%0z+ow!YSu1!rU0}J}T?Kl6YfDL7yB9}<_>{yFIf0qPZ(WzW z)a3P~TB3zxrGZs+cAysNcYpNr3QJiRV!`pY>ZB;;Q9PK0;VVE2**fhLAgc zktcYqDk+zz4xwH7eliunC)JmUk8v*|l8(Ar{zVgTt86UWFycxOYGBKn7%)ezNU1@> z8haAXw(aS>l`+>*mT?gz%Od6AX@uCM5ugNm)a<#*g}KvyP^^A+r|8YVPkdJuQ&si4 zT?E;c4CK1;`M?@cJo@R3T8Z*Y^j*!!LGo$E^N5jDuD6xLD;8Qiz;0hVir%Uh~HUYj5 zIg2K#A$FOwu7InI!6aYYR8Rd>YmOu*h11?#uT;hj+a$}&2g*W}BTE+Ryje{D&z6I% znvgMPk|BZia4JUBAI4a?rQ=Y+@3R>DH_Vhw>_#A4LQ7lT5Gw@MrVBOw!S}T>)vYg5 z9=6UMn#rd2q|m%ncAgEQ9Gk&2tBm2zA@?0}?rO;~qI=4q3W2t=whs{vB>hV!k>NOQDuwDdLBER~rD7Sb$DO_=S|u+{wpn>NkqQ{krFya)8o4#;j800FN z1TV_t)IYrm3Xi=+L;$m+$;N^i$fBS@~qP^Q&qJ7(2QUzk5JG7Lglk7D{N#jG^mRs8ghkE=7uAi`sZQMO==2$!-kGjPes04RJ0!HW#O z+ZGddw_VZ{h0jG!w_jhih!JW_sItJBIBPVBoy`W@$x12_@!d-+X; zF6$@1&^93_ccmsSdv%QcRm&g}sUTfkJ;LKu&!_C@ODex+_xs50S&yLIfLRDtX?XG% zZ4lD9`gK1A*|Q2@^s?sJH97D*IXquhn(HI}Zzfyc*H8FI{KdGhL;jw(tBI{D2~&_6 z*s-TV7P%7p9yTiw1v%tguZM|)+E0K2&Sm8N1c;g-C_V1wZFJSxsFpT)W;luw1u=elJoS6Us!2Azt`!6#m za~pG~{{p?8!kn=AJ$l2ZJGTrhv@kJyOAIL>+4!rfI6D59l+~Cg0%!n{CreVuKFuml zbpa;93hzOR(kCH)+|?LSZhXZl*{W57;e@3Tvmc9*>(9y#JYE$_b>$$%=g*IzK61Hn z>@9TBc=YZsB%PS+Io3xL_xKQ`?bwr@GKh%^gey@6GWSEK=GPyivp+n(ybKfCX9lhP zV(En0$fYIgp5NsTg8@y6t+hRTu-%;Qj_;@I+s{k*v09Ih=Nl(Co!bv{Co6qd)7>uv zbLC`I4Q}b==``M7Wj>4CQmLNB5T4ZI7KA{Sjr`>TZyKb6UII$+-M+AGf^F($7mCBp z<^^8$2#67ovrl(*S>WbvB0ddmj;mMtJI-y3x08wtbJF%2+N@bdh+>j12XC$V@#V!8 zQhx84?_0StDNCf#6P+{$jZvf;L}NUp=o9m%ozD+9)rWPuvY5W~1t1STvl#YV z&gu1XWNC7^B`7~2#^$A#$@D@Ypm#|f@I-cyijrMNH=rl-5STCBpyTaFfHWkBy7UYn z6B^)Th(!A=(LhT_Jtr!%?$*u(Qc(mgAez#hL_E?s$jI8?`Z@?)1mpe7oFVLJJNANG z^@*Oan^7q$bcmZzh04V5unM$?XU1UZaa}l?Q*IV8jX8;D1L1I$K_1>?cP4j=PW`jmAHtP=!W072XDr^t4k+{)A8@EFwknENY z>Y3k$5=%souiesV=|F$v-W-T74t};c(Rs?(0@GGcM2j7yZ!tRM54+0_7a|&>i zxD*vxBnp=3=JggA@J6uwR1W_2TlRZCxDaGOVL{?zdcW71^j>H8`@*%* z-GAParH+I}GO53G&KxGTv(R*dpu;AG!~amrjvnqGuS!ckyW%kWGGPm9Ais* zJa5q95N1(ih@`J@NwlJ2ok(kq$(k%%jp2^d%m?U~GDCQV|9p7@psLW0$HVy=EB{x} z@Q{3d#t&NDbes+)1c|3&D!a)|>kzr6s8?-l)fk0H!pyYavyV#6`8*CD=9(}Hs(uR6 zhmcm8?pihMt#mi|u=31gGy) zzhapuUKawoNT@{#Ul#mL%GX`2#k}OQg|R;KM;8*c9YnL*(e88hCPp%uu&(rgnV@P73rw7c> zG00xhs%o}2g=UIQkK|3%Xb1GCs`c|O7P@B=Xb#U-*P;fLHNv>)Q>)c1TP%d;k+-mQ z>~I2=eFD|tz$Y|s z%sYMe}VbfDFsk9&67?gSA)`DrJf~d&YXSuvqJ)MR~KGP-YVrHJ3%$KkWDq&Vh(9#SiW{N#8!W!t>Lv@0rv9Uv8ZQ9&hUQ) zp?{m<|9ZIU`S>Ws|9V~E_X+yRki6-fZklNFj*p)xj%CA!z9X~&ac|SMCiY^gYln$z z|79X=ZPPQ~fLd5do8-N3Lmi^cA3*b|E1QOmwY4b}FDn#oc}p&Qu@SEs!*^kR(`Tr86B^_8b!zDC`;FDQIq$lz1R z6f#<8Tt!VYvLV%EIFc*MLf|yF(I?QH!k(QopYf(|18E|fqbR^uW66+_Ne#pDHe49~ z>jt~Np5(nxvMJYxU-!Pg2Xc2rT< zObRzYCo|Px^-eP>AzO7mu1X>8?$LCZ;oaeKQy8yoo@3#+RCmA1E=M=xjw0zBlLF+{ zzEiMS&|^h3NtdOmmi0!afVoXt9iCVdy(tt*0Q_ZJb(>BC(9r~UxM5v(s%2leC}9Hw z>bN)l$o@ive|%fUbXl`dr=-xFRzOc82_rs6R!NPf2ymxCjZdI41nH>)UUwF> zMF(RpgX8>){=*O_Q0oGcD&L1=FWh#8u(38|TcW;}Lq;%;nblq@D0a4wZlAOrlVG%& zPWL(nEla1YkVc`C&AfG%hfZI5VBgcnX34N;amyztw^lShUD{*MU(1wpWHpAx5CFhm zi09vE?(bOWFEmH;&&!nm42%AoVY&@r008$tFrBlLxs@Zmv6Yp%oujeie*xEBXjIc>$J4bUV&BlO|?h-VfKs_d@wDR>~B5|s{Z04Hz zJG2s-rG+C4lP}AiV}K)5f`PorP+x-xCUCZ)Md^Lj5MQ+EW_QEf6oe?}<3S&7$mN_X zq7fpQU|p#3t1l-O zbV5&Z7(HQ`Qw<|Nh|t(A`6GP4^v~Be?;T@h`CV(d&!%9vv2C(2LW;n#oG=*nV_cfS zBDII}&*(i5^o^ar>YlCYzaWm9vG}hDH!m$Nl3}D+x~ueY5gV!7(!zhVTsNoLqd?lu z8IK&fKm5S%=0R*+A`9sa2Xc<9ukvK?&gOhvbNT(;*ZWf9uyr3~>qLk#zZywk68q8?Y)|)WC&JC-)*(ay4G(nTw~2c3II#(XffOD;WA8O^S^U<~;xtssg(O`6-x! z&SF*hT)8&6XKFXtQuc3RT#B}GjU+3gr^->nT$`%jp1j(LKt(oIoqQn$M?y&8kt9fj zBm>*i*}|uSc6u=#pB~6+jx~f!n?Vx>*PY$;Ldj$v1gYe<1+fMLI{g^dMZs#fC6 z0LIqi!uPJBkB0;UQN%KhlO#TCrfl(5?AmPkPl)9|zY&>VASE5Z zl*K~H<5F>vL4oeMJDiQPAd`A_1VslO1W!y!Ju^=6aekvxLI^5RN#>l$&r2WD>T|!p zxWw*cZjQEDaRQz}mb2O~hg>n!*#0du*0)bwH&0v3hB8eSrAN9c+~F|Ojfh9b43~y2 zPJmo3$PckYR!hXZvDW2rMHaAkM%5Z%X>oEH&_|Q3d43-CAe{vcXRDVhco?CphJY~L zr{b?y11-PEAOO2=_-j8%p!!_$VR(XddL56H$yWK=Revp6|# z6OhMKxsG9d+{6IFT3G9^7@JrL4h%!p5|ry+!dop-(gmd?HDEoe7z}k8Qo*@P(87CY ztI8fHB;X~HSv)8)UmSiuE9A3b14EO4HSu57|t^bKs$UA zWd5^b3?bGdXF1qX(L7#-wwkAotk+MLH>@KH@_L+5}=*+4~?(TGgh6r%m#ZNvk$O?>kFg_ zzq^_Jv6p2uLlsI`Wt+_+C<$b}-*IFERAakkQFJ%u6y0`N^n;3CdWHH+uvVi?$BU_9 zLAE|*)0!jj;UqH!Y8G)fddBP8@?g{ zptmqYDT3Sz&?wUfI6@$r-Qe-KOBPruauyfFKf0q68PRwi$0>{^5r zqN0-XfDR6cbk$UxRvCM!JJzNoh*O%#*1bJhr9v7(w2je3xxsM6U zW@Q~p?U;)r!yEt6%AqBns<^|3!R9q~a2__Uf7ZQwEH46!!|rf2HAx*PV0<=H+-5Rw(5wZ8Dw&ZUT# zAQE4=h|G(IIq_?uMJFNck_%1s)CxP#{yj`sMZno9@k^$E{{obqIYkZLq0Rk?n`K~d zEjMbK7yowRZC7^`O0uZ)LX9-KYY4TpwuHSpe%8FfX0n<(X0$T>&Ek^2e@l$Kc=<;o zv027$`m>jxQ#z)fd_T_SB@NqB>@SxULurnomW%~v4Wl()9xd?_Pr9<(ri?eRpm%HC zC$3H>r5;y-r_KIdUH-<2skR;sjnkT%HdGCu*GVzfAq6ubf3bXrk$|5ZPPT9Dp0>Pi z?_H^y4w?bqf7i5Q#MwMm=BIaLBRQ!Iv#`piu&UWc86dZxeNeK0leIA${>1n6=lOc3 zgDL-TgCe&GYxWyh7#wUvgmPEq?M6-%BTq2=xcP9KI~G@D<>|(!AA26mIQjF%((#DV zz49BdGa6Xf3(4xdtNv8S=seA{z1ICD zJmz>lb3E@2?$0WBcDcxVpf=vXZG8S(xqda@^;XDx&ybHFAfI1BeL#SFfKXlk)%tJw z_1X$-vlConBRs=GyoZ4L2m|*JCb#{o_1}u?wUyXrC%MK(dX9#87X$Sn2JS%|^7>cn zzZKSNE3(Z_bd8Pt91ZzC4C-SD+{5VXgz&dd{#!x4wnE$Ngx8q}&$1Rb)V!~aDk_d| zLBZp0Ba7LGk!yTAZ^|?(owE$;Zgb*s4jty^)kCvUu8RdGO7u~1cU@7o_6F}b%tWoUg@Y#Lk#*uQHSfvZR(pO$NRmJ>|QR^2*Kgi8v!SR@me!7q zZQk3954-)qu*SzNdqRvqvN*u6D4vRpqNO{>k>3p(&0;dHkXXi)VX$EGXv`jIB+f;f zUL-%SM3=rgr+xqDzBQJH=JF*N0N^i{^v^ty;-7fnzlao_?mtnf|7Kk94*CD|pZeEd zurdCxEblj~|Dl0w2!GMQHUR?k3Io&Z6{hG&8`?Smm?DjK52$~igcFSj8f$_>!tm!+ zCoyS8!ucRBSF^;ugKJmn5htD!m7KvCOKFlcB}s8??zOAL=WfO1{cDk0fW&Y&zf7_T zS2X1V-nm*a^6VL6n?bxW&vN-FYOya9YtJ?pi>^bB;BnP+9EskC&|s_9`dh8RUD2|B z?T^MJ%rpe4+2XG54J<&NFR+rPA~T$nMd~Gx-yssOAnV1AXT0%bRl&)ROK02&=lqN_q46yvfS))`%H%jdx5q zth)}-AK!e6g0T<>ON5HJwMoo?e1^9Z=8qZ> zftn*3+z*)~oTI}jR3V*OfbR07=ZQqwTA=zBWocksPxI+Y0lRSX+m&j*e}+)Gq9I`A zrF!hdFH6p98+AHB@Yo2@3Z(aIRH8;YJyR{0rpnO={hjXe0yYAcN<~0ZJ6+%CGo2~1 zlSSHqEU{ymOwEst4t1_z9Lw)jZghQ3fOK3%WSR(X`P<3*j-5oqeM(|?`;})16rppq z?Y@Xj^m@cLx(FFa9>$xFYF-766tw08lNwm;g2s5F*y(k}yuSwMuNB)BUf0Ov{s%36 z^@;uxw#`s$GbaIGIGXV;#Q$tm$USK{RDEyHDYQ)DM|^#tAsPehRV+KvtJ=1L6~! zef3peYz0yer)$mTJUOZ%>^zE8+1)&l+6`(D9Y{O)x{tfAP9&JNdQ&?EB3E5?7 z#Irn>_kjzpmqRF0W^`8iFvLNY00-~K$$}Dm3aS+qRf{uDd6jJN?=p7?oceu>bLF=7 zJ?LNS+>_UHTLEs-o%jJ3;0e*YE;p<1UGVCb^t5`}KFDo&W|iw?r(Uy{k;@NgSsZE3 zlpPeTEJFJp2aLt8;AuB(Ec1jhcHuH-5Oz zS?F1;dt`ttn9{IW^Dr{DQF`f^_04?&U{2K3nSS5 z&9iptMlLU!uW$nwVy5O|%D@G`uClaEah{kU`U=U_T#y5C=jX5OH{c}=r~9pbwndq; z`yUECRBj6OmW<*){Fdy`LP79L3$A!bizSUZN$RfB`Pn9NK+R2fFg)MH8v2$ls-=MIYcNIcs=QYeUq}g>#a*og{9ktac8gD=PU{E@#bryh!8@-7qUU-vfhH}BS zg6i$@QX)$`Q8RjQW5-c2_JA!VuGZX~vY*`Qz5}rMW=CC78A3c1*ntLO2pHMI-d5OiLDP1yFd*i{0UORsb^E zFz&Q;!M<~xk1Hzm?7`_y{Kdqju3L^tf8xsq(u7@;k?T$s(K|B!I zn2)EpbF%LFyPIc-oFv5jl^Jb|9e+2djp~4v_+S74e@xB4H>m#>zU?0d^8V|5_Kh;vw3}pOQz-D%+Y_|T{7jR)KW5)mg8id|9r28;yA0Dn#;k+ z#uA2ng8XyOKO&KXX#fZS@P}LfUL61JJ^w$%@n5m@KR_P;T74q*m9N0~m-qZOcEo+y;Gh%LDII1rwceX74p_%;N`3Qf4b$XomaF(~sb|-|lkowGWdr zi0Bql$5}~*{i7IY5vxIwOd+UC%9LdcDKb@9Tqtpaid8E1tUwGD;`grj;`9a(4@V8D zwi!mIGG4_#3cCRf(~uAr#u(8tL?qx0YfV$4_;iZ5x#TwPDUJ?mM>5&kd|(iT?9w$` zC8rf5w;+GL4rb*|*mZbC5$jewGMZLqyRqI(f(ruWV~SH7Y93I`w1s#Cb0O_YB8}t} zz5)Fa%aEB$Pm3>=|HHL^isfJV`A01OuPXnqSol{pYb*UZE_9HcB+Iu7C!rVgG5JFdCXjk1E?{+7FQ>#Fwb#DZ| z)N3NO{itc$CsKcgzukH!b-yPob1qtQgVEu=23xQ9 z&^Rc3M29o-+k5J+hL77R#&sCL{Gk&UASV^MP8_s&sb#eqtJ{)4uoc=}EdS#*J)%ZL z*I-##x1x`uhMF+Hoyg2dFgG#Ac8GSM#V^GSYB7JJw3^Ji5o2{qIIsY$+EN;B&h#Q3 z-yVqh5hEg9Ea3)N^xy>hC`TAKy`5hokulj((yC(UrPmQaPP`wP1%+1UJCv)Y0F;}) zbEHJYS#)v@TCc-*O;u8Q>!FHbA)R?+^Rm0|HL-z;uRJAVzJI$@_@UL%FaITgf7tWy z1(58Eae8_(0zz_1>h%AegLHcTnuq^kA{dAM-#bqK)_-ei=lF%>|G$vHjz4z_U#aD6 zy&flj(ayRxjz%Lb8^S*==6a$n1rqtqLA>9@WzgxKEV)5MkozuQY1)S!&*F%DFi5vE%#NB zOd9U=j#UfpMT4qf`5N$=m|MhDjCi_>)-mzaZ3#vG9Q|aHJQz=s9weM5{`8Y8{tWgJ zi%>5rIa$dhb=5PDc~ZKD;KRinuNA-EoPs|qn>wW*2>Mig2)G(D^7h!Wx!S9A5!L*|0W5B)H=v#JbdhzP6`UzJ%@-oCEMPv|=oN_vuKBfb zNBKG&Ya`4Bn!Zow2U~|DRWCm8=P%5Q1U~kcx3jvjrr(Zzk4;lu-kh&5^o!7FL9{6P zha?Ib-J(g~^aayv=jL<4y35O{_vSb8@BIY1(0I83Ma$JH(<_^w zikFRi!Cs`>9t!w!s2VI$rwm1v7`EXM0*fgTVeWu~Y!QEuhLYu3&G0)76pYAQ&BQ}> zJ20$sfE!2>fe<9Ba73(w!3R`YLKbFPCTd=b*c zK_s#o!AN`}AKeUpPr}Q?gqdExFp=xB^WK$=$+)M5-g@`zz*1=@jj}i3(`8?=f@ERq6UjB#DL>nFJ-8 zqXKq|;@u#bG7-_;)mbjeS)jLI1n`!cVRYCOlq{qPTYIvP1>PYL6WMCj0d0x%#D@NX zO+0k10C!4-s()g1<;DVg7{eO&QaFo9>B@cY$c=XDmWx)|0@+j1D$1Oax7|a4BTG)~ z$ClOqL4g(dZXr1IVayX|y4z=O&wy2R38K;SJtUHMV_9`bmc&$ z!%ngfGwWRL-Pn`kR&0dey4c!ztIqz@KAyz6xAz*%=Hz{*XWO13y0nWg8p)3(P1=Xc zXdYgSSoG0s#2u@nnGdoUnJWwSt%n8gB_9zYFTbxJp4k52!&>|?bS~uqekj!0l_C1= zngrxMG$@ZR>${M=+3=1r!xH|0g4yoFm|1xW_S?%DtF*k{ebaAKGdEi3Th)iAK~r{v z98zc7I^2jmzCX@7%JL`T3pfD4A3x&X`%3>7H{frs2A$r&b~gS4T&2GT!`Y7iJ&ODr z{nmCy2G;-Zl>UbZ{hZgz4rkG2^O;(dr8>=~i5i?WT%U3_mX->^P$N*d4OH2HVXZ?- zmV=u)J8eII7Dz;42%duEd9kGxs#)ZsJ^otgmm2E(<$m-fm9W zO&THlzn19J!#m<|HwZ#jNTuo!zatY{a3D z2)M7ZQWC*6II#>u;j!Z7FniHO2|wf%ZaOMbuj09I%?9vT(qq@3Hwd^Szhr1R`I4H0qCXe4F~2=IpG69)2xz1uoE zc^z8btVvDz*?AS)!LYx5ljd?w*XeROJoufp_pC*)+jYIZ_Y#HP_^Qf1ATVN}+H}h@ zCAWk3Gyk(f136ZcY+pfu%)ze@W)zInG0x;AQga853Rc93+W5pvpE{zkzniU(GEV!(%S40EfKcG_5keK$ZKD`V z^lqRcJ7%6`U)d#KNJ?a}-KN8=zDK%R-9a8V3i86%If}&xk)bw6&La3|e&#cxsxm~J$9?`H;!PmDr zwRVd}B9XZ5U|L!0e^=;ihq}}ya{(zPiR&$hjlKjyWOV>2rTUc9{dPO<$%-{HB(&<6n&w2a13 zPh;ksdv($7O=dHhqC~LJ#cT3o(WK9%TueXNqS0;RLY=yee#vrj6_KUJD=pRnYMKZN ze&lJEH(G(}c!lc%9tEx54Tk6(^p2)$eK8p)B^EJMU4@aL7(NTFL{q_kKA`d>Hg{I+ zI2Ezi|JV`nH)0i=+wdKfZIBjMfIY8BF;Xt4b!hi*<8bBUbqLB#Yjzg4?oRMdUWAhU zi^q{>)^4Lz;NKvz3_sw8Azzz8zL7~;3ZWm>R8=JX)GE(sztbnEB+3Yl;>Yl(gsY9& z>R+1f3FD1h1_@yC2MTb({3sTMD_K)$lcRu3M_#fqRai-8p5M4h$kX)!cTI%&$mp(u zUjV5u$zbCOl;x)%vYmy{R5Qt1AXk7ZBfG#-(3GzBixu1_tQ;DO*8%=+Xo-BE)L&)^ zWG#?78AKdQy(!rwZF%lPjk;17&1KSIfE*xzA!cM-vw)(-HgSB`;dI+*>`)~Zm%n#~ zN#IIgyoMdYOX5A5t!TH^ryvCi@`D=}a^R@vXc;sI$rpri4%y@?4k>ZDlt5iMYzy9e zH5aduBza=~LBHWAF)=9MC`LF2rh%PLK3jETZ+@7dH;KP%&sR@-(_Jz%MCPLMr(EtL zVycIeVIuv@PR5`@yu78QT~KE{^p5!nNJ7y)It#c)VP~kDh_YSHMYV;1 zG5d%D*t{W(*=fr>+Rt*}bW@o;mRt-X6%<2{8?IVwu42>^|*_KnMoh|=q7QN6*` zQz%lzvc#E9G(;q{4^X_{oiiTH>CB!-m`*@MEc)Lsm*g9#VKFo5vAXGUEXyRleDUX{3NEZ@>V>qGMV3C(>1SB zFbLLQ%hh-Sv8fc0v@m2S^G*V5CX85srXqM+w|F436dVEw8T)ENt{ZB+la=v!kn{CY zt?Q}jlk;{M8-5qr4B8;h}?~PwWLl zEFXRB!IKl*#Kegjs9{x$w@1)V?kQJnc91X&un(6XnIBKqgLuiNTg5*Ls1j{*Rb+9e z4=NzU)yWGbtMjF`L9&?dZz*3S`xGVT;yGUunb+rD4{kx13f&X2vQiF$t4)+@j;2q6!2Zecy3Z$X2-#vxPT z1tC+MBN6%hRxY%_9>0_CAz3y>)mCZ`AQai-9If?5bQto6EPaT+M}*f#m0a5TNT@xz zAM5I?=KG}F*idja^Ya`Yf?YRj&*;=9MtGBi$&uGEcI;UR+DL(5ecK*LdXaH9nE5_6 zfOZ!sg*FY<&i#u{6G-dQY5&?>IpAKd0aWKU@nn+!B-z{0@duz&!z~}R#mgX*Ja@Ym z?G5Gc6blYQuF23Nq+p#73c-#wXUy3jz$lcffu*xI<=gRbc4_2L%NzG(rJBBW2~@>F z%!@QCJr)6Zt`S>W#;qy1^gEidcm!QejX^=3lFD3BQez|{6eGrfCaBdM>kKwVJFUmyFZlcWXYIeehqF4!@2J}X zK;QdUm8ftE{K$nw_Fn2i&^4WlY@UT5_F_BEhpFp>Nu0|aNAaOBLekts<*GBK*9C70 z)7>bm&|(VfZ&rXTV&>{9CZ{G@B%wee~Ls5rn)}-!ZG*yKXtlVD&+= zf+~lyviJg9rwRTk-M>jQhwJ>QhRUN7oBG;Nk3b^d%es|W zPUjR7Yh(drA#jBT95Z#IH(k+J`5H;s7L!_IsY z19I}(6Ypm;KY#CdcDt~9?l5BDxs?^R(U@TX8xLKw@wo5!`+(QX1 zMM8NVzAw6T1Z*tU^)a7tzjhuD@`o zg6cFU45O%N1r3Ch4ARNB2O+P^omcqpGtNTs?_-04R>%(T^SqLE5?a7vQKBm;uQWQb zW@pC5`bO} zxy}$r+!Da0E|XQ#v#D%^dVbM}si5bbl2$TDQiFFDW-PuuRDYno1t?G^vzl{e-!!>J zdZpiVX?5~Y&Ca*pR{}blsOuZk2i;AUE2`)oXN<3FGXvq}48Y>;3XCeC&-k&S`7=bV zzIm9u&&zOJRtz?3(k&!5kitz{+_76=KBKnFzbG$KzvkDh%7q3-5j$2?eb@;nq7<>7 z9L1hV)cM?VCP6?~+7DP6Aq7Hc#J3n+z3Ot1z#o!+2bFwoH(n zjLmojc^XDUN_5s9hqX~BT?t;w4X7OAdxX- zkA91C)n?*$%;2}AWRAs090MH|ELlCm@^yl#ZP5VkY9zxX{_f0sli>b604pEho*d@~ zqwc4v6<1q7#e1|(%#9R%IvQ`m^MOv;~O&I$$9jr8MF|2oG`R5iQFeb8ja zG^EOiN*Zgk0^*X6;DI19V>_x>UJ@=?Z;f^?o{kz^FVn)g#w%3%FgSnAnIO@63T4@V z_XlM>Z1!OI!P8LAiuu@mgar;!$sG?UmX{fBl(4eJ$%uG-MB-67%XxVGv+FZSX11Y~ zQm6J%^tq~m#F*_F_D^B^-S}W}p%^NIt6HuwuO$T4vSO|)Y8MbfCGbwmU$1jg4_4QvQ5zOm}Dqr-B1RwM8zR8bk zxi|IB^=wmUmpde#&hpTA)H!v<$Xx=#BOzdB{^B`=)i#_vl2R0~-^QHo2`R>WfF%UK z)_FSQvi8ZJ(YeDOAdM;mT>1H|$GfYf!cx1OC0Cqiee|4E%-%fevRNquk50}~qn`jf zi8EJ*{wZEz4Us@MI4o3_)7f@Wal?;2E(hr`Sl^c9rr&}&g3bbSxEwgM?wEqrsFek> z`zXS3LLlD5MI3c7W@i72rdz*V>{G!Ice8dz>>$w-L7bU8zb^y}?YKsESIHxGu*e6v z%fvyz#epe#=&b&3tQ#uz(^O3pkG-{TrNF}Nv?SLf35cqURRLDRiI>Drsoh;=PHNYw33zA>`9Y<8M9*{?l6skJlBBL?!ib{Q!G;vox43NWj}iM zjlU*yf^kO|D9G#plq(I=^e$nf@F}qLvyr`!Ij`5@7Y1oMfc&67TlgL-wB+onEJl9+ zh1k${55E4KBCo+~vI>CO%~MBP@ECMs=wOp{A~|obyAiSCoX`9q-*<{FMZIVz!Fuu| zaXhqplSB{RpFn9+fYVt5bO3<>YT(`S`wYx59prdt{R?4uxBT$@r!PCglE7|kHBocJ z*3*;*>D#ISUo(l_3o92;J64QYepZ0Jnr4`a%>R!&);^Iw`ZGF0A;u9#YdYa|Z$@ulQzs??-oHT^#RwU9x`9b8~K=zf(=p>Jkmt3~Pnn{l!P*khdnYgO1F&`8_!k*;1a%rhoSQ837A3ldb)R;)sB1>EA#M=2@ zO&*uy+ee$Nkyt6E&V>c|$T~&~uxF9-Fh-a(Tb`L-)(ysnGq2@(O-)OH8Akehs3%S{ zO+od(9jpi#C2@pT`E`mB`Jp(|y^(6R%_QSx=URCFRI+nyb^Y)RG3v`a=5k??)dy3v zC(P?&eixTU^S!Rq%kq6N;;DW($dFe%@*H;h^{JWc_q?L@0&*{#^(JZ5NX6ZkCm{pv z;-={ou8h8?GXU{C9*+8ZtGk)oY1tK{sNvbH`T>vs)r#^=Y* z<@3ph|9$}M`Wu~QZrSTs&46R#s4##bh;M~Y4=Eq<8nG2Q2(}Wh~Tz zgTrYt94yF+m&_SAG71m5V-?3ztpoTO(RXOkA!+JGAXV~*xNDiQuFK}(;pRSZ_ph2? z$lnPq0UIa~>85H2JTuXrGp$0u9&ovjfHho?n#H~++>~JKuFJ9l2J6LbUsP^A^PZGD z)iO7gb`Il6&0n6gqp5RLP0i7mcox3vqj{-ig*XXof07X&y^>7y(B&3ArE9UDSDpGgKbAoDrPFf#76{a@tyKM;;(V%SVi(@_>D}oL^eq0t+Dm@@ zisZ3VS(yv(jk6Y4WiV~B^o-YT29u_|9(M9H-R8Y>f>-)0#P01g$m*@v*TFkcY0V2U z_bL)RIEjhny<@izbwGu8&lju@oHV~_+9}&TO-M(0LH$&XHjV}9!q74Yaq6Pwn4UCi46hIgZ6;MkA^ zp;7tM;32fP3&!q3gEMbi8;LjfI#psfdi65U*R0diayo5_8^v)r%yX@V zr&7X5k5yR5MaQ*23+Uh0y94Eb}R(z6L?)@+X!ew(FSz6ocEh_4-%J4=m!qype=S;WtAtOA8t zT1^B|M{8>e1YqQq4Zh64bLD>}BL|%xNY)^nB10YIRVgllCJ2PV4$O?nRji&abhUqtOWDeW^mQX zw0;K)*AqS6x%(3XRJPfnODn`-AdHlBP8dF*&Xb&5fMd4D3#O-|B=Aq<*Qj=U{c2(4buz_b*>33zF2g#<9rFPAa3h?T?CKjg-n6|VbWEzB?DOoBNH?9D0|#u#z5dp6LM+^RHC%ujZS+S@bqoihZ(YZGM5b z?RbRDg27lL^k2AhFtd--&o5U@3*6e94B}iU?6C+M>Lf{P(oOqQ#Upx?r99CbWHAyH zC_AIYP{H~cyV+0vG~Tbcbyq>Q5YUDnZS*%qkRRCK1x)#kd5}$YP{olQ7{IN3p}9W} zc_NKJ&l134LSmuxXD)BuCb3J!L3y=mvIPh3b>7U{tC)jdATbb?5I0P}pp^zkck9pN ze`@+P{wzoAYytNcdco@7jQvm(cp>ZQ;gAd8K)3q))aH|En5$F_0DwP9j(_HH{+j;y zAB(R3PafxAWi36}Oj}H4mhS)eAf+2@yLNL~F1xAX0-&bA+L9Wl}|y5)C!b!r9$y zmf`0TXrPbY4gkxFV1iC@3Pt4%O_FqrH(dkPsnYDU9SaJqX=_Q0RfEt|56g4id}G5E z2Hr@zN8~n6~4 z{pAy3;O@0&SLXK_cOAP&`MguMAG@+baLASzs~FK+XXA*hLIF&Xnx<t2Ja=EA}6F#Av99)1lp~J8y=J) z97jrBRF9)zeNvW2HCo-4vU@ZPkKy)5bxD?G%@<9s`Up#pT}INr4d?Q&qRbt`E8UYJ zo1RGD`cuI+ zMvRnGbPe<(%omLRjLO&+S1VP>!<*SSRlVrf%z)W{-ap` zJUafWa{rs7<6mmuf56f4*CUYZE6M#|C%*sf==d)s(EkQG`nS5C9kvL9-&FAu8YORe z=xQjGikZ*8uhKcEN|5>lD_cknIdZfwD_CzizIn4a;A z_vMZ{S%M%h8JgyW6=Z#dGg5Bd37Q{YMw*38-QF(c=$>5LaOY;DF(+*SuTc(~G!V(-^? zt;>(SG+Vwq1>UriclOS-K=27xhZk^gU~@eeG8m65JT6PxLGel?^X5EDtA3g?d=qP+ z`>n5Deb)wlmX|R2m*-zU=lC#G9++nj=rqy#(TO?Ip}QZ&z}z{-a_V5`v5w%uO-!O; z8C;o~3u;;*z9`ZYiWt$Q%%@Jw_EVFJ>mVb>gmjR{W-4pZ<)zIy{?nB3?CTcc~O zClGIQQnG0#yuE($y-zSj9)wL5I-coSY5g&@R)WWd^ie_yB)-NP#C%MUZIRkzC&jkW z1Mdjd*q(U#-st1my&Q-A&>(TT?z!-=ag9)(_<7T%J@FaBk@5ztT(Um0)a7;K;KVUo zcG+!jOnJtOW)YCBa1J2ltuZ?$ZmyV+o__V4r~AEdL`d{Rh#l2t-7Mxn(DK$eV_187 zWMRl~I3O5gYccs~&(&haC2fVRcyZCsczF`5W{pcKoQQsa`@sPUfwmgpPw-}1XzBo? zA=!=C^5moOs;e#b-X9XhncuWAwxUtMa`wQj8Z-rW!lwZQe_GWEU>i=BB44$fPnNco z0Mre`C-{G7*>QFq8+K@{YHa-GP8Mj?S9C>)FuEl?h{+e>uPICd(IQ2L!2K5L&>?lf zaAChWArr`fPRQ7Z3Cn}nH^%|TMe0{%f1@Iz+9J;kC96Pb9DmC(Ofc7<9N_Kz)c%Av z)<7M+@5$)W6RG+uf43;su@M=Bov@t3uzWs%j)9L|fN5lWJ6a@LzA1zEY~5IQFZe61 zRfp0{<6?u0O^Nkwmld+0;-Ro~HYCX!b*A?LkGv7LkCqWr)4z`(mVERm7aN5;;%%mrKYKmK|W*}G3Po{D>?YoeT zbeR53*3lb6Uh2@O#CeWzJ5n9vF!&c5#m3_J4(~zcmaC-zo+a@jx--X;x-hG+I%%@6!V7WTUhC^Dw=3pV6{Bg2B($Td3!@!LD#R;J>9HNF1Ef`O zdC2^7&#P1BF6kY9%)%AxFgsYANL?bDJ=d%SCtD3mIRsIYBm zUt|ex*iG0mr;W+4vr=nFE-OFrS2eu92~p6{Xepa31lkuv3^DJ$Z)(?3eVlVa z&&#e?D##{^T3(a~l1S+n(2r2ny6<2J!X`1!Q0|BllJ3;~U7A7n zibzkbb%>XElWE8V7L^IzDScM)xu&6qE&f9lZR|wLTH=}^{wZ#L5$IKt`rfpdz3F6j zac%!5PST~0kaC?R5z?|;-K5kqwL8hq!-REOc^;3-y@A9246U%*{DdTT1~Z!q8(l2_ zD2Y=_=gu;1HF7pYq5Vc{HJH|SBv_`Gx)bdto~0>@h^<)e}G@`SN}$j9RL95zw~kZJ*2QTwzspe{oj<) zpKJcVW%SMbRQT$>1>y1Q+9#jBQ!u1ywZ|#- zyk{3KVGD6GjvT+>&BRe9!vziwXK#&@Nz)G3v*$LW-N{uD_ixR-1`NgHLgQ}Q)el|{ z=ck5vyMDz(@P79AquLW<-J+wEu`%1*LO6?2Lph-JUd7+lE+}LsPXv5Pi($+!* zZM>w#lWO7DLil*C+o}yWr~V)qxnht`V3-u~O^|fE`~pTY-B#vGO$qgN{%9dS`lMc_ z7d?;hvY1(Pni})CWRzLEfMoe8rS0F5At9Z#G30i(dU5r%X2y#{2F*frJXPI2uLI zYx^pKe51f4wVymB@O%=DIcc2z0bb~SgbSeddjbnmq0O?a6byKc|8o{FX?`v-kQ|P5 zxb`b(+ki;C%o|V2^2qy!hSr;->7B(T4Wt4zuz<5R#AR8Jx@^H5T5<7D)ZjR4OyJUq zT85NYI8~eVNv*5_X{L@j+r^+mK~c#Ij@8u0Vm@lp_92pqoA(tdu92zoFHo}{$EpP-@Tu5{T0*Mze7n`Y3P-qg%A917BwjGL9x zv1aTY0OOoWd{iWn0C*b9ngir1*0?WsJVidHB|&KZwct7Z)-*NzWTN)l>Y?%_!kK%5 zva@f4mBsoJ&BUKeMlMd%&^sb&5bc?q2`Ipf1W~4>ATiJ1 zHrj6TDbhGaU@`IL6Z4buas z9qi!8!d`J3(`%SuX)d{CqJqps_`OqA@Rt47B@{m2@*fg<);ODkX>w#T+YKhT6+;GT z&qe^gXOH>R9zGkyY3K6%9XsF1;fx_?1-wuMjS-J6tIgV%RxV06CT?#TTnJc4X9gA> zvLq>vH+@(eaY-S;%pzl6))$|8w8*k@eSzmy}$U=40T2JR?6l@j-BFa^nd>oC?CjR*zoTd}^zY|Ngp%MT6y z`lttjG-(Q&pA5;pF-*tscGclX^c@#rZ-QJeq~b&*6gsoYsckz%kpYUBmx%gbO$w}8 zG)vjR`3}*k{g&|*yZ9J^lr(EAM}YRqvKokc69Di5y$cxzeEAxs<3F(t;V>c3Ht2`T z5{xNe;VKyJ1qI@$l7LCH^nrM!-{i#QB4Ab>0?!#UuNF$F_FGkN+kJ08){wiHc7U6a z0lx!r4)-YYYr-y3X(9$ghX3e`jUR^)SvH^S#o(Uz_6;5Q@7A4avSGsdd5Gw{G>jXFV*uoo5+aQ5EkqNRE>k26rCi^xTQirO|L7Um^9e0s5)WKzx~;W+|lxCKvu~ ze7vT0X^f@I7@Hd#2DA*tf3LH%sN+gA>Rnp(y}?)2e60->(aHdRA3xA{rD-n9e^A zfFz9&xL$ymRrm%-9uwIH3%SwOd&2nP|c^+AJknvzK|af#SpWvd8|W zJqT5e5}()36G&G^lqeq#Rx|!#{THSjiM@F-6x|qnUeBu%ljU7C8)GJ4SuFxHKK)lO zjCn80fsot^u*?duhL)=~q=eYY53T&=O-REN%6GrwVA)%$*tsKh3Eye zY7Cc=g!!7o3Rg%;YDA?)fHz7X{cU;+DBWc&l$eaJE;ngpYc^ASr=}wc(|a=nz3~ZG zosxHtE;IJ){ksP`J^w(L1i8FczWuZSOqaN)}o&u?7Mf~c9N&@ zFDTLk=8rrve;BYBybLiS+0|hB4i^sUZ&o3jm+U7z%3P2%a6;ZBfZs&C4@4qu$5rQ%{$JV^JhZV^18#AS}sj4`# zohb?mnxMIZH8OF9ZArT)0@5k06T1|&%dqAFWoagYs%kUs*-p2rUWczybLmmED}R%k z;^dDyY;yXwIv{pyZEv7fX^_37A(!PpFX_fRL+V)JO4}lPANc{MofJ-tWIeH|;6Ss{ zL|t+}5l#WW=&=g^O#vpU!58N6wx!eUk>Y@z47?mX_qW4~M#p|0nwUoM6t9Z{?VELf z5`Oq@;wU0Of+5XKK=j`A%Cre4I#TdMQpJP#s3!$V!^R#uK6jp^Mt;GPEPvo%V98XJT};f)fXjfk=fj=3;HF zI);fItlv8Wq$#syJaI+PgvJgg^NNr{NM1YaoD?z5a-h_6Di$aQHBM`Yk8O{uu@vD5 z*D4yo>ncx$@jc|$NWw#TG39+RMc}@2qg;?lKM^Qs`ORGGQ4`JFNU2Q-;%FL#=EW)p z)0hrpE_O!C6-)r3?-^0sS@*GjmW4t`(O7K~mR6i_*o=3BtMyT3fD}^YRK$zz%DazL zo)(j;K5wDFCC12@y3{nvZqh%0b;6;Wk~XR+i2ys}$}FNh%?3ewCi!5ev(rtvJ29VJ-qfD)9&`oIWv@M^JgW55{ru>Z6Ut}~ zvJYq*HRv$+nNZn}GqWO-FKc5;jh}Qu{=zVH?=)p3tn%H{@;CC)>@HDW2Ry1XKch$W zBv~$z`fj1()#Z`eZH!k%3``{^7qgx}B)7IJ2EVr3*GHR(XO~-`(7!4jAZzLsKFQ!k z@rE^@+}acBkjrX zLe9Jx9yU=g?GbpQ9**C)x)ATfq;yczt$yD2pn9xgQS2LVSQoPS3CDw(PL8AR*;K!p8p~t7An%6Btuk)M{JGaj7A8{AsR4+jX z=53uU4}JI7%;ou1gfR%Bp&N63eSLi4Y@LVg9+k4+u*5W+jpXr7u1jBcw98BBzVY^b z&b?T!-kzwno@&sswB9+w>uArf7p+CGOB|yrO&_4EF~7b>mg0v3``qSlia%Q71{Nl8 zL{4%UCL;JXZ8&kY+g{BRzDPw*xS1)DCXi)}6x(gZW0a>FN2|5e*Hy<|-(*jGQyR0` zo;T>C(UYd4M{-@P7Wsl2KTe8Br1u}I?>LyK813@RrB!l!uFWKJe22C~>v;x^d%xkr z2RU^URJFyX9n@Nti)zRR_KGrSc(fszonMOkP_&|SmhAN@ZKeqz9xx<@Nfb#b^}0HRp5;%5YgY%O0-%7z zW9yvGxu=+9RyQHvYW!O@Hk2kPlvJSvVRR`8jh{0S$rOJoSt9i|rmdkB)YWf|NYxXL zw9VKuop!KP2(hv%V-GfMz=)1~nBTd!trkkV*^7kQYSmweqU$^sTVQG@q=?vA0nrKG zZ|lIStl$gHc&~lfC!&K7jlj>Sn0eGBG;1jD3>hb*d!$8=U09sT9~+~!k_Yt(_;DO* zla5^T_105vq;B$UJY9qW+>+s<^cx;gIF}|i4x|QDS(NE1EZybSD`kGeVn_zaGjbX!$N=_Ng#S?s)~;mTgT0aFsxJ# zXpyD45m17W7PROXz9%gx{)#f*5ce4R#4C!?wR>SAip=|IPd{tb^ZT~8#2{hoLIEem zI%+EwsaLa2a;K7V z=*WSprPrTZLO1xBsMmh`aWrjgW>>GgGc7;p$5<(zeWTg@Rs#xOIj@tISHcP*%C70G zX6^4WJ)Pf!->PmLbV=Mu+n^phtLB`n70W8$1^n!gBP`9)Zl@;N zsrde%=B_)Q>i_#+dzHNtGO}l8$lfy}BSKa;dnHO-BQirtGQvg5US(!wWN#rvb~hv` zBfopATle~2rSIqWczi#v9vZUB^U)<%N%}8?(moz;Fh%e9xq`U}VHM{++TMmmgvzMLf?pgcP6tf38A2UC_ zCrM>Fmrs&vkx*OtqR(<{f_c1-JA{%XqixzCAnJy(M%>U!U&cthh_|~?Wu6be8|21w z#v~vo=3QfdVlk#vJ&O~}QK#oAqUOW18(P<=b7*hB9-fPd*mj=Uz zq>6X6yN;3iQV4^wMW%eb!hHJ0o8I$}j?@Rl+~l`sCDj&4b5fUT?aL7PoN=O*Cfoe7 zSKd_t$F$l$-cw>>x@upYpM9>DvBf^3T^I7O_`dVrJB=@M^No5kpT>GWYQvIsdaU4= z$1ni^m~;8%DG}Wc%u~W{&VP2j_~vza%~5-xf0@sG2F*hrh5JkoHyfM7xJ*cltWzLA zvEpV@1I{DI<^&FLO&dvho4c5pS`;=E8%MaST8-WAB!L&1u+9#6GFx^wr-%+QNNMt( zt~pxO;~zENqR%$`X8m-M6KiWsm6h$W zWbZ9=$jYW!F+AXP*<*1#8=ScPmH=h|5c>F{MW~8lk~-J`U@fZl8vsgi{>${Q#f+eT zM=L9rL+RhxUXS(_Fwk|4UOnJekA{6jQ4jtJ57%VI22vKX zT+UDj;_~{9(+aIGyNN-Z+q+6O`rG&4cm@#bDUeo_4=x#&Z|~MbUZx7z;*$1`OU-;``y!&+VlOYEkNY@VpzDTJ1U31K3cd6Vf=le*-Ph>4MX>xrI;u$|r~yBprM1za^Ax3Ql4t|b8>S0#Bw zy2&tvT@8g(FAVG~rH9CwO<08)(_fCk(e~xnv5)e-6L0KPM3Klj6C9dw?DCt(c%hi} z!6GJCj~|yCQIPNGHF3jwx@euG*(mr-%3o9t&)8oqf+X9>%y3mS&M%+=GPlUs$?0W( zBk<;AcE2?r1694*U0UaJN>&V)DIa=oi5D8a&Ebl@;UuUciDQi|+TNdjy}4B;4?9gs zARE1JG#7mTSl9K1P6u~ML7+$t#`AJ*-0_7PrJ(Z|vmg9T!xNfyPj?1V)GKk`&yH6j z%Ipua>bOH;Dmxg-48r!L(p}LM!PPSwxxv}cd8(joU{8oc*7;jhYO40Ji@uiTyM^+{8V6*c!8QSTqdL}HdZPoaPz7x&1s)R z@NSLT2B6(L}m7m|36O{5< zBdwD~=O&Vosh27WrM8zNmKm+%VWkyc_Rm;-9zmzh$t}x?&7N*Uc#Db|t(yb-O607_ zO?p5~K-V)|1(&CFpL3k%V~&4i=)HY9>uA7@BX%E+?tNBy8h9?0;6!oL3sP4O=2c%) z8CIdVuv`e_Q+#4Fkli+}|6*+P-j2%fTBQ-u{F{5PtZ|nF@}#oPv|Ue9sHJ+s2pm{EuLno?f`)LW)F{mOOY89y)bsXiW=1A+o7D|U2Xn(u zZ;5n&+?ewi>`}6KZ*r#eE{PACAa!m7wb|I%9{%wM3DpZ-N+NHCcT&~Q+Ha4(X_J_6 zEbk)H9#!<35SLaVS#sJvJ^p3&%2j@8c0%dW?#j;olE5zojqlqe%1`+y6FtZ-9lP#c z-0vr`B%|^4iwFc9y5i*7pL_#n<p7FYOx)Hcb1s0vB;5 zLpJWTT{4{-0czG=w#Y_rFs+&rjvl°8`HY327?j@_*IoCt7cK%R>w~- z9QpuyZxXbgewrqc)cXC%+q$CB+|ag?S0Omu_3K8d=O*vy+_2Fnmc4X5-~??m=@Qc_ zp7D}}q`*Y2Pr9)2RQK~ZW7BtyxK6i~qD+94{UwXo%DsiyGx8_MI$fwR&PkHdY~qwssqxQ{`O%Zs}OQ z&g^+PeV0d&cfkb(7zF3n2AX2?I@<1w+cmEb_8sH9x1PVktZ}-h=FtN=Twhr%#jl-8 zx+gaDhDJ2=o?w(2+d)LfxXyA}l?KN~0A}6riX`EORq9lts}flu4<&PF5|`BKR!_({f3J z@@?_0`fFV~3H^9xQg7o5h%!zj*hy!VI9v^8sC1)S%5}#RAawn_9$-R~BYW<&T1aKV zfc2a&ss3~Q$Me0(p^e=Wy}UX@U)`7eG^&qpQe`Z%w<{;dQ`Obwu?7JusZ6j~iaCHJs-K^52D6#t%C0m> z>kFRDE$9=>Ax@stn96vcoQ~uAJejQ<+i;#CJb_CZ8@o_hIXhPC+4|COdF5#-MVF}l zG2nJ2kBnQ+mt%v@gkt>$#vCgq^&ybxX8(%(ytgYQp*|D&l*QfU!h61GTJ|^Shx)nA znJNalbyK%pA+G`+>(Wx$I1fRfA4D*JHsnfxjFs{PK8}1GHqBf#Na4klV&l`j}A3L#r^XPK&dZ%gR#Iy&fPN9_k_F4ZBmav=rTY`+M z_!1-li95s`O(YrI;0ECb^*yfv4gD&`M#W5wX}Cr@xC-Iq%DBOJ;Ta0#L2ld__jC8Y zHPXz#Ew~j7?O_PB;=f&+C`|odwl)c%M-@EiVMJF^RzXvd*U=g&|CRfn=|8=5>>@)v zoIJd$T-Z=~KzkRdWUL;AQh`xASU%*42OeH?)a?FcvQdE|Is%=yo2R>nt)mC8hqI%@ zpTyIynH$R2t-;ZzQ)s|1Br>Je93+Xj#oa$ga4623R|3A(2=Z&H4t3FANI8-mXs=73~MP)h&l2m z-IkcRpX>1?eHZ2hhS8pmzT&fnjLp=7rB89HMDc0w1dT+iP-$&`a808b@TW&_jAu+> z%1|&#aal}W`q~R=uKbu{g@N+xpIf zK+6_gtFBjrmxA_mNq<(P(CL_JiN#5^X{;GP753ya^16}hj7F;3Q?w;j^%?8r-mkH4 zd$x^WiGt0%m(D3-Da4eQz14E+U1zF0|13GY#E*24TTHTzLQlAIZ&`?QG%0buL3kiw z_Muz3V!rvwhq^uI<}4;VN*=1^`so4G?s1B?GY_z4_jK9I%{V4GeVWkZWpSZT2xFa< z$~Rq7$EO(^Y&49!k$@*JP*Kj8x`hGrg^tO6F$HJ<08Hn+I<(q zlOH!{D7MaD7J*3bAd7&B7TW&!8|iQIfK&b#$OF+S!z_dhSsv~l&Tf`=HvhmfU{Ce# zUn1WH@`Luj$wKyDES3@j0Q)S&(Gutc?`;UbTts0ZnT9`Kma>+SLCfz;+il9_DITB+ zqgd9lJ9g8O_|mN~JazC%-766ad%j-etY^bckyC@3KCZpLdj0)iW|zui1KyPaO4fIg zgTl|L48s+zFjWaI2z=ryhrmJn^w<_<1QK_e6jyxM$)EZgl#@LR) za(A*f%Xj1h_F6a7Et{G6#y8^+GGDY$QS&5ve5wfe{+_~P)2DGIO@U<%(cL>@WSSZ! z0ze*7+$O6FqpTda@!ruAo+FZhJUo6Y^-ie^;2w_~*2+`OyH}Hin34%DRN7^JdRkm~ zDQt!-h*b^}H%`(fA;`}kk3p+?(v2`*%m0>=A?U~?Ig_eq+S!+&2#b0p(RigIp<5f5 zGR7IjXfZCXUZSmFHgz>1u#Y}5%Loynfn>~F>1w(gBUH+1(V7`+D>1CD{G5L-RW)N` zsJClms0CAK>PeG4tFC&tJ1;?uY!Ktl+uELE7l;d(iKTEH^VW6SGSTO&eY7hFzJQcg zYMu~Nm>K(Lv_s$e(kFJM$eIMa?r2Xj5e*Bt6m;+U=Dldc>ZKJdQRTa&&t9z;>TCn$ zO&L>r-CK5B&y>!qyMdN4HYv{F)r~T;Qje+JeIqOo;y~Km1WcnyV0f`pm$}QYq?M+3 zfu*&%HNpR3f#LCJI-#wpmD*~L1{*8YthL}<)0u#rmao)Q;Kv;__MF5=7Q5zI3l{nt z$-EaFG5t)E7XnFXsJ9{}L&p2R-g+mHWK-sMEr6(;uP`>8z-_LU%Ea(k%6mDJj3Z64 z9kYvL$z>1uhCSPOuFjIYE|RX#X_8M>JaPXCchUK@g}B_5EdCAkIl_Tnf{5L(@#$+M zTUIWOK%l9;89r@7Ytr4~hUdFu&-#S322RbmlqrzwfG_Cp5;S$IR)`lJA9!k2m@8o3 zx}`Zu-@MiD%fOVKZDsA~e|m+#vCDuVO%vmv6oUy84RBeKliOw&v$k@x9s_0eW{z>X&~(;`0YGE-&4MfkKthtmjdnu( z_&fgg=s1&RnuKgCg7uaXiQ&G~7hNxN#WX+7jgTx?pKkcbRJz2(4;-jMQ+EugTo7~r zsDu6(OHxdSbU`eVkWmYiK-r2XDseeg%!>6RSBNlzS|6Y*G#1q`v}{{7-}q3qQ4YHG{$cgw zI^x_@IeFnMkoo6ooSQ2VTdGa1*Qoa_UV1sqILyurJV)0hvn3A()>my7wYRqvZI{kB zmh~vv**u^fu8wD^|G2P1fu#YoD|zS7d6au50d(uU>@}v9iEc4hS!MO7T614$^7$lk zGhV9B*m}njC~4q7N5Aka!@SWMZ!-|>wreW*6tQbP!IYv-s|A0cJWiYXs6R~*t!rY2 zuTrvZ)+A|?TJk18@a3H1Gm)|cs^znK686_V(?!$?+_Bn>iC6)D-As3W-qqG)bn=y; z$M6%q(QTtBq6O9YT;7~5kn^3T#;j=^#j5&>fo*S3#n-B$nNRXlaaeoo*E#ks(lK4n zuxVx+R@Di4xX|+93~M;^B35^U*rj{@_SGck;`yH@$c2Vg3PErg9*lZ6*WRz^yS}x% zyzqeg3&sRt=H~YB$ZF}!BU(9Dj*EHcNozr9#0oQFDMqvxH*%@#at#cama!OnUCuaB zeNRP13%HpdgWfB`Hke<*#Qw|WgZEBUyU_nQn%KX0dO#y**yZStuo$-I|A?*-K4AIH z4w|g%Xg|gedz;Z!-4cBJ`RW^C{-4%&fkoe*r zSss2|Dd$aQeY)%S4Nda>?;RsnsZD8n0cOmmzqOIQCgF zy(LakoIMoO62!t8=kHYZJ<3u&R?e8WhH3oO+jZMhFW+hC0eM5!rVAFs-1?VUypD?% z9pULav3}Zb^4MB>dT%Q{QhLzKp%ue7{AG?+%k4^b&mv(9jZ>o&dgI4=X&>P%OjVbD z(s8yrbMv;1#O%JClDB71-?_rxR*7L)H>NhcKs2s;3c_LM)c|QG^uucA(5g#p-CE=J z@hB%ex;(-x{pkBak2KH5qqYH#12kd8Rn%{#xtUa0djZN5%^BqNuA;|t)% zG@9IXv|IwF9{Qs|4wV+R50afl4%)o;lbRc7uBS($XUh9z2ClU2xwbkmmx4yd&BL`M z@Y8EUlhZ3#Jsy!v-%V%R3&>b{@sj;h!L0m;W3aOmJ*nj9ax?${W&sBqH_|&w|7j%9 zGKyc%Ji*Q6Utr>hN5Bcm?~jEb%pM*v+gRDU!^1>aW{hR`=q)|-{GJ3w>Fq-6QpOaj7qeyaG3gJ3=acI?IWEvD?4&Zh z6FFK4+h#F9tp>~D&3a*D_wh}GhG{zN zT^{XA#J)_dLPz5Ks&|O+4Be#)sC8~I$to!6p9*fD3agfqCKDCF>O@hd(i=l=EyAR%RSF{tf`1rDnt=+@sPC6ee18kXW zvEM$aauw&ZbZ-g>%Yk@g>f9badT6Uyj8Odi>t=`d*p}5mD@w;aljXcZJ(?BgRK%$w4>Oh|-8SadL{;vt64XE}p_Ozct6);&a0qZ@VTj z);jAkc0jm5;YMZc2T2>^;EEaSr?MRrr6o9Vj8@ARR*U1KbO0T1UQzKD6BUM{wY=;! z6sTD|@whyBwuJ9!!1R%gJFyY3!s&}cTrsy~^m-y3#yy2tne^W9EW|CxjrEVtCE ztckm>Q)a*2IJQG@)?S1QPXdp+VrrA7CfI^xH0n0JM(`eOWXRO{+tlKz=UX6Mf%(kpk`xJE7}}>;Ikdt>YN)_FGK z1zF&~OwQ87W+Wz@qC|WVy%AHPB}vHq^BuPQ6ISTKtqxB#UJ>Uzt85fOhRatA+Ry{; zMff#nnfAn9dh<;0D6V`sfSbOG&{uB3YhBouFBGHgzUvD z)hf#}p4LKWr*pcvnM;;Jq!O3$tzYv!&tOtc*1#=^e~<&l2>aL~lmjYkf7cSiqG>)$ zi_ZluOZf7LlPfiA`2xwwCEs2;Sqf{=*!cw38__1=DT173>7pQnjHGH zaTFn=`I?7~h^t6Oa+4zqB-6u>S-lR>88EPZrXN$EfMe=-UU6$Anl1mkOL@7r zQ9z2W7lp}7!%V6`TlTAOyt*VUgc_lj1mDw!V4bD0SYX+h)2e^ z%-Hd>(eVnR4Jgs$(&woKaVm-AggVl6*eXD}1~wUV4AQ!1Kp$$$W?h<$xHN@5GDxO% zLY{QdH9XeU8@xMY5yb1Vf|FvuEyMpttf({gdX)7>d`Mwz233ZT9V;=!{@Urp7aK+5 zrxps|1Sq&3ua`R)e@h_*&OlC}n3~yk0MjSa7^KW^*knt)GScerg{(cA-)oMT_T~ zlo^U4+lokHEZpphDd}gif^{}QW9qa#fMOxm@}x&2GXiQmK2dmW`6H%RrUdV<`Qbea zqS%{Z%ZL^UzH#M|V4H@cAoa!|L5)2&JsRVOrz&X6cnl@)YeW->to^^pzG$Y?m#kHj z(LQe<*+c47B=^}5Yt4v4!8g;cEwKEi7zS%pex8OrC*z(IFc)F>?MQA#tk| z5nqhov6?bcOLiZ6ZyrHw@!GB`Z&G-ZMm#l z3_buQuXcFdk|ma&e(*vxe?%n8=fs&skM4ldo8yt;qD~1t5hEuRB6saX!KwkS&OP?0 z&+$LWG?BRO;QiFHOxOHtS};yX!)IQSF5em{reL>+pPq@Y7Og(!BJlLso2zSHAdIKn zRuQo_4~w%J08WzST#g;TN|nYWGh81y7?UC03aW4-9N$tBJ*N;WrikmFIH$$NWQ+^q zs)|h`#4V#tHnJeQv?}L!hsv6AaxG+9*Dnd%>)Cz2raWhn`k_s1!bB(5D`yyTqf@$Z zCaW7q8X-9le)O@m0ChSyBQs}Vk1@j?w=~fiXR|cIPx2}qnT_g4x!22V8mKw_Zxlj{#yE6*ARMpShI z?DGk`1hsgz{pr?U}M#ZGEZ zZ*E)vU+N(%yGVjEVHPUgj z=Y>&aTRA`1-|N4Jb@5qQJkDi~q3$YM9w#E&hd@^3r8>tkHbnjW*ur)5=xY zUOCsaMX2W+hDKB0L&Twd9F!OHUcVJAanYtBFYGkaOwdm?Y(O1-#$ zz|@&2 z+17;ygRdsN9=)fdY2$w+e!r!?{1{b%-->bH{pq!Suq}r~ZSIV{i-JdYqEll(8qk2v ziD~-em9HH{Y@u;DM}Q*&~$CNa5)zCEY?$5W$`bG0kyfzp^Zr@TtjE zb;?4q@rq=!oaLOSuFjxNy@GyjK&D^EtoP9;w`LjegwI&$UyxK0Ra=bonUlL={gi9o zIaj>UPPUJ+tH(Z=>2!N$c{@Z>E40!eGHym$U(8*U*~=nx;uPD)rEx*JFC&|gwKR0y zP41#^a;6^L&9iGU){(o@*5IGUAM4=~NaM%TIA1VW^zjUV7bu(9*T>PFR-9G%IeTIL zrd2#)I;rlKfDGM5b6}CTY1l^YIGt^&Rv%~}Xn^iY+{i5OELLIX#>uFFHEiO&`JfB= zAx2+&Ei|N}Q&Shw7#~q$x1@dzw%HZ8)QW3LwGqJ?l5c%apBvOy4Y?`325J|-@gns0 zc!A;-(}`pZjLOqKd^>OMMSK>&NX+99#o5C?dCQ${HvAsR<2JRZ4a1-rI`3Ct4)qR2 z=WWBJdWf<}OCGS^!!&?Mh*^H-gB?aqcfCreUP-pYun*7jF@Hu=9D{FO^B%tXhHqAx_K1yM{(vukzD`zw6CTX zSNYuc$9pSYkgYIZid(V2f=R_GXK&qz0jfMJFqXk;MmW%#5Hw!uo z#Ii0V-gv~wS;OuSCU$lRFW{xhSU*ok7PG1%+gGoCzAA!6Ri_WN+7s@i^F8W8A4&6v zgUBCfuv?sWk!l>okK8<4Yl!88T}I>e+;>=N`-C+qPx|cJEA)MP>}qt>Y)f><6!1vI zs{wH`(dhWjh89EYZS^i=FIlLho|cU7a9rTFBND|REMIj$&xuh(qiX<+Tjp;Bn;f@# zteI0NvDmNWoS`Ri0q=3FMgMkJsj8>HK$Ul5`jA7QIq_@F)Qs(xSEcVgjt}=NGdvbB zxJ1iKKXWyn;~1{v7pD?lzMZ-iFk0u!0D?9az8=o{Hg61^o#JYb$6Uq6W%W*UY(3hq`v9 zKn(9GwO^oM#>rC}$!iT)&o=iC+?9REvhHLa^}$A9#>&CnKNh6KJ}x8!+&Sf|J;t>c zfYXJBfAT_k_wcop&7%)rub!iJl=iLQ!UX{WpqDtuWmibGDg#`IYGj>bq-uiXHguva z6^}59u9IyUeiA#TUc+`5jpNht^?>mdSHUAfx}76g^^!8jJFHh&XmDaY7}asaI60>q zyzi7`YqC>3)VJo;J@r+C;Gsy3&x3)}TIX0rpHnvpN6S52)T&rsI+--Qh|hErh34gED&dsN4N7uYvTZik{D!HcFLI^2Ei<@`jarGBYad z(e-f-+Viuq-)Tw@V9471R5aaq+387mdFGvl+vLG>j3vd&-Yd(Z^GgwYR*FQ*XQpbz zlldGbHu^>eS6Ua@J_em@W3FI3-@O&&;CG3w)68(DOhTiMI{a09H2TJPo9(kCsa0c> zbKsx^{R^ybpHXk9e!knmD#X=Le1&~bnaQLrr8Lc)@iJcUbWu>m-aMYkU20B-DOr{j z{jamUee3x)X)RYFk?qqxlqsx^3X-@%uzIgsk=G;)3!zlh?;xMX+6NlL#d_X`J zP^t)f`;P~?k?8+s)^J;~)Wh-Tg$jEYS~81B9}J=t`Za45f``rh5A%n`wutkG-FCqT z7z7qUm_PhEag?=HFgCw|0wcL!0kB$7_<+dTKx;MO9AMW}NJ9Dpi2X{SM5GKVfKvXc z1UL(5p!K6J3a;qi%pQKLwl zk$tX%49C+0=)cR_>v;CPga9r^FsKX4 z6A>J|Ez8flR+y)Q?aF`Y6JVFo@Bs!{1RjX&K!$&S1>5YdgX=iapdVOBLAPdDWdwYH zL2NK^UOxH@ zmO<=3ST-I=ioz2(pfAm@z;geVM2{pl_k@EHq2TogFfi#&=R9n_1kEAs2?GF#dpQ^+ zd>Y)S8x-vO82Et5QZFEbf&uzRkt$*o2?5zb$^S>B`aQUVar&F>3AT*5 za8+rApb_09)GyE?-~fY8LR)GeD(9l7l|58H`>hUk=7jz1tCvCCo%TCBSZ*9Xz@UcZ z0|EVp9f%&`tA~AsMzrkAv!a>FB4`5)T>phuo zm5B=goPy$jB{AUx46@xu2J?N(13Cf!X;cKeQ-BXJ$PE+ipoMfW$3p-UDlW?zgx(Lm zmInZkgZ|T4Xh(AEpl|!PV)zBC>S0BR-0-njU^fY@CuqgL^jA+T2!!H zEIls$LtMxPX+rl0T&Oty0T|3}*i=bH@Ivig9r}Z;FQ((DfFWG%{g<&4vcAfpTu|;< za9c#d@DH$H4kWlY&j$$AmpatNBkRkT9u=%hmhS(*`qE_l11?k?4*@Lnd|8tY6xcWv z7~H|&^$#9^bCR=ZzM1ygZmaY z**d#9TK-e_fgEpaaUFo|8$(`m)GYr14Ytk@U!p@c$VP4y&`>@(_m{^;WCzlq_c0r8kWVI5y1$2og5MuN!L)_gMi())1qQu_@3xO#_*3bg&uXz> zK#omb*!?F0lw5veGw_ojVy~xt051L1!QmP!?w1Xs?H~i#a{f;Mhix6Tp9^9`9Ap6H zE>T~C5y0f58eD` zKL^APF^HFuV33n1+F!M)*-sg9?#P?Z9+Wb|+`m=7q3VV68@a>eyT4-qtl9=XAhMNL zeoX%`one0YOBhkanIq-CKzmT4`)B^gPBDmrf$hqO9X$}qgF&TUKj)0%Ci{S%F8j$N z&K`MxZ16^y{g0YMlo#1BPJbzUSOX9EfXD{;`~(6^-F-{i!9qV6QIzdG4psnS0S5%s z4ki3k0Y4!?c_6X)!`u;DWx(c)d;u494QcL&Zp85m0*wAaT7ZBCp!D}c_@3W9Tw@Nn zzF{-pzXTAoW)Rss*O8FGvOb4vq45jK{^fv}@PUA4p$t$%`N8Y3T7hq5h+iWkY6T3+ z@a$}M^|^3a!RaGv1rZ7ubRy^=D5zNd#=zjL5WhV|-Z-bBtPm=G|LR-h-vL6- zN_@SE8qg1RhpjJ0WC(L!h}Gf9=i3({NU-46hN=ww4=w=42T@bV=i5_vPy>TAJapwb zA~e{+0 zg8AE8g;!huA+Xwisz6+;5D*AT^`~1nbT#*Wbs*LUBOWXuAklljW{si{_z!c3sT$F` z5LFEUJxD;9``^|Z?}va`vWXa-fI&w0(GFV~yq_rI+!5st2DLvxm^-ieA!|Q%)Ulw+ TZUEpC^j|y&002rtgZ}k@p6$^% From f7e7f6e2576049022c2665093f66ed1040950471 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 10 Sep 2024 16:27:35 +0800 Subject: [PATCH 11/23] =?UTF-8?q?=E9=87=8D=E5=86=99stores=E5=88=9D?= =?UTF-8?q?=E5=A7=8B=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- TODO.md | 2 +- ecstore/src/endpoints.rs | 5 +++++ ecstore/src/lib.rs | 4 ++-- ecstore/src/store.rs | 6 +++--- rustfs/src/main.rs | 6 ++++-- rustfs/src/storage/ecfs.rs | 5 +++-- 6 files changed, 18 insertions(+), 10 deletions(-) diff --git a/TODO.md b/TODO.md index e405b8d3..f19a90c3 100644 --- a/TODO.md +++ b/TODO.md @@ -36,8 +36,8 @@ - [ ] 详情 HeadObject - [ ] 对象预先签名（get、put、head、post） - ## 扩展功能 + - [ ] 用户管理 - [ ] Policy管理 - [ ] AK/SK分配管理 diff --git a/ecstore/src/endpoints.rs b/ecstore/src/endpoints.rs index 6f781704..9af5b963 100644 --- a/ecstore/src/endpoints.rs +++ b/ecstore/src/endpoints.rs @@ -408,6 +408,11 @@ impl AsMut> for EndpointServerPools { } impl EndpointServerPools { + pub fn from_volumes(server_addr: &str, endpoints: Vec) -> Result<(EndpointServerPools, SetupType)> { + let layouts = DisksLayout::try_from(endpoints.as_slice())?; + + Self::create_server_endpoints(server_addr, &layouts) + } /// validates and creates new endpoints from input args, supports /// both ellipses and without ellipses transparently. pub fn create_server_endpoints(server_addr: &str, disks_layout: &DisksLayout) -> Result<(EndpointServerPools, SetupType)> { diff --git a/ecstore/src/lib.rs b/ecstore/src/lib.rs index 1371d1f1..81329685 100644 --- a/ecstore/src/lib.rs +++ b/ecstore/src/lib.rs @@ -1,8 +1,8 @@ mod bucket_meta; mod chunk_stream; pub mod disk; -mod disks_layout; -mod endpoints; +pub mod disks_layout; +pub mod endpoints; mod erasure; pub mod error; mod file_meta; diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 8fdcc2fb..487886d4 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -32,12 +32,12 @@ pub struct ECStore { } impl ECStore { - pub async fn new(address: String, endpoints: Vec) -> Result { - let layouts = DisksLayout::try_from(endpoints.as_slice())?; + pub async fn new(address: String, endpoint_pools: EndpointServerPools) -> Result { + // let layouts = DisksLayout::try_from(endpoints.as_slice())?; let mut deployment_id = None; - let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?; + // let (endpoint_pools, _) = EndpointServerPools::create_server_endpoints(address.as_str(), &layouts)?; let mut pools = Vec::with_capacity(endpoint_pools.as_ref().len()); let mut disk_map = HashMap::with_capacity(endpoint_pools.as_ref().len()); diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 54aeccac..3a8fbba6 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -2,7 +2,7 @@ mod config; mod storage; use clap::Parser; -use ecstore::error::Result; +use ecstore::{disks_layout::DisksLayout, endpoints::EndpointServerPools, error::Result}; use hyper_util::{ rt::{TokioExecutor, TokioIo}, server::conn::auto::Builder as ConnBuilder, @@ -66,10 +66,12 @@ async fn run(opt: config::Opt) -> Result<()> { // }) // }; + let (endpoint_pools, _) = EndpointServerPools::from_volumes(opt.address.clone().as_str(), opt.volumes.clone())?; + // Setup S3 service // 本项目使用s3s库来实现s3服务 let service = { - let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), opt.volumes.clone()).await?); + let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), endpoint_pools).await?); //设置AK和SK //其中部份内容从config配置文件中读取 let mut access_key = String::from_str(config::DEFAULT_ACCESS_KEY).unwrap(); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 0117cd83..9de4d419 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -24,6 +24,7 @@ use std::str::FromStr; use transform_stream::AsyncTryStream; use uuid::Uuid; +use ecstore::endpoints::EndpointServerPools; use ecstore::error::Result; use ecstore::store::ECStore; use tracing::debug; @@ -45,8 +46,8 @@ pub struct FS { } impl FS { - pub async fn new(address: String, endpoints: Vec) -> Result { - let store: ECStore = ECStore::new(address, endpoints).await?; + pub async fn new(address: String, endpoint_pools: EndpointServerPools) -> Result { + let store: ECStore = ECStore::new(address, endpoint_pools).await?; Ok(Self { store }) } } From c930fb45f1e6736b83bdbefad0bf8c2b4ce33c61 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 10 Sep 2024 17:23:47 +0800 Subject: [PATCH 12/23] test --- ecstore/src/store.rs | 33 ++++++-- rustfs/src/main.rs | 12 ++- rustfs/src/storage/ecfs.rs | 159 +++++++++++++++++++++++++++++-------- 3 files changed, 162 insertions(+), 42 deletions(-) diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 487886d4..5116d562 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -1,7 +1,6 @@ use crate::{ bucket_meta::BucketMetadata, disk::{error::DiskError, DeleteOptions, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, - disks_layout::DisksLayout, endpoints::EndpointServerPools, error::{Error, Result}, peer::{PeerS3Client, S3PeerSys}, @@ -16,11 +15,31 @@ use crate::{ use futures::future::join_all; use http::HeaderMap; use s3s::{dto::StreamingBlob, Body}; -use std::collections::{HashMap, HashSet}; +use std::{ + collections::{HashMap, HashSet}, + sync::Arc, +}; use time::OffsetDateTime; +use tokio::sync::Mutex; use tracing::{debug, warn}; use uuid::Uuid; +use lazy_static::lazy_static; + +lazy_static! { + pub static ref GLOBAL_OBJECT_API: Arc>> = Arc::new(Mutex::new(None)); +} + +pub fn new_object_layer_fn() -> Arc>> { + // 这里不需要显式地锁定和解锁，因为 Arc 提供了必要的线程安全性 + GLOBAL_OBJECT_API.clone() +} + +async fn set_object_layer(o: ECStore) { + let mut global_object_api = GLOBAL_OBJECT_API.lock().await; + *global_object_api = Some(o); +} + #[derive(Debug)] pub struct ECStore { pub id: uuid::Uuid, @@ -32,7 +51,7 @@ pub struct ECStore { } impl ECStore { - pub async fn new(address: String, endpoint_pools: EndpointServerPools) -> Result { + pub async fn new(_address: String, endpoint_pools: EndpointServerPools) -> Result<()> { // let layouts = DisksLayout::try_from(endpoints.as_slice())?; let mut deployment_id = None; @@ -97,13 +116,17 @@ impl ECStore { let peer_sys = S3PeerSys::new(&endpoint_pools, local_disks.clone()); - Ok(ECStore { + let ec = ECStore { id: deployment_id.unwrap(), disk_map, pools, local_disks, peer_sys, - }) + }; + + set_object_layer(ec).await; + + Ok(()) } fn single_pool(&self) -> bool { diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 3a8fbba6..94cca1cf 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -2,7 +2,7 @@ mod config; mod storage; use clap::Parser; -use ecstore::{disks_layout::DisksLayout, endpoints::EndpointServerPools, error::Result}; +use ecstore::{endpoints::EndpointServerPools, error::Result, store::ECStore}; use hyper_util::{ rt::{TokioExecutor, TokioIo}, server::conn::auto::Builder as ConnBuilder, @@ -10,7 +10,7 @@ use hyper_util::{ use s3s::{auth::SimpleAuth, service::S3ServiceBuilder}; use std::{io::IsTerminal, net::SocketAddr, str::FromStr}; use tokio::net::TcpListener; -use tracing::{debug, info}; +use tracing::{debug, info, warn}; use tracing_error::ErrorLayer; use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt}; @@ -66,12 +66,14 @@ async fn run(opt: config::Opt) -> Result<()> { // }) // }; + // 用于rpc let (endpoint_pools, _) = EndpointServerPools::from_volumes(opt.address.clone().as_str(), opt.volumes.clone())?; // Setup S3 service // 本项目使用s3s库来实现s3服务 let service = { - let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), endpoint_pools).await?); + // let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), endpoint_pools).await?); + let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new().await?); //设置AK和SK //其中部份内容从config配置文件中读取 let mut access_key = String::from_str(config::DEFAULT_ACCESS_KEY).unwrap(); @@ -135,6 +137,10 @@ async fn run(opt: config::Opt) -> Result<()> { }); } + warn!(" init store"); + // init store + ECStore::new(opt.address.clone(), endpoint_pools.clone()).await?; + tokio::select! { () = graceful.shutdown() => { tracing::debug!("Gracefully shutdown!"); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 9de4d419..f7a50bae 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -1,5 +1,6 @@ use bytes::Bytes; use ecstore::disk::error::DiskError; +use ecstore::store::new_object_layer_fn; use ecstore::store_api::BucketOptions; use ecstore::store_api::CompletePart; use ecstore::store_api::HTTPRangeSpec; @@ -24,9 +25,7 @@ use std::str::FromStr; use transform_stream::AsyncTryStream; use uuid::Uuid; -use ecstore::endpoints::EndpointServerPools; use ecstore::error::Result; -use ecstore::store::ECStore; use tracing::debug; macro_rules! try_ { @@ -42,13 +41,13 @@ macro_rules! try_ { #[derive(Debug)] pub struct FS { - pub store: ECStore, + // pub store: ECStore, } impl FS { - pub async fn new(address: String, endpoint_pools: EndpointServerPools) -> Result { - let store: ECStore = ECStore::new(address, endpoint_pools).await?; - Ok(Self { store }) + pub async fn new() -> Result { + // let store: ECStore = ECStore::new(address, endpoint_pools).await?; + Ok(Self {}) } } #[async_trait::async_trait] @@ -61,8 +60,15 @@ impl S3 for FS { async fn create_bucket(&self, req: S3Request) -> S3Result> { let input = req.input; + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + try_!( - self.store + store .make_bucket(&input.bucket, &MakeBucketOptions { force_create: true }) .await ); @@ -87,7 +93,13 @@ impl S3 for FS { async fn delete_bucket(&self, req: S3Request) -> S3Result> { let input = req.input; - try_!(self.store.delete_bucket(&input.bucket).await); + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + try_!(store.delete_bucket(&input.bucket).await); Ok(S3Response::new(DeleteBucketOutput {})) } @@ -112,7 +124,13 @@ impl S3 for FS { let objects: Vec = vec![dobj]; - let (dobjs, _errs) = try_!(self.store.delete_objects(&bucket, objects, ObjectOptions::default()).await); + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + let (dobjs, _errs) = try_!(store.delete_objects(&bucket, objects, ObjectOptions::default()).await); // TODO: let errors; @@ -173,7 +191,14 @@ impl S3 for FS { }) .collect(); - let (dobjs, _errs) = try_!(self.store.delete_objects(&bucket, objects, ObjectOptions::default()).await); + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + let (dobjs, _errs) = try_!(store.delete_objects(&bucket, objects, ObjectOptions::default()).await); // info!("delete_objects res {:?} {:?}", &dobjs, errs); let deleted = dobjs @@ -207,7 +232,14 @@ impl S3 for FS { // mc get 1 let input = req.input; - if let Err(e) = self.store.get_bucket_info(&input.bucket, &BucketOptions {}).await { + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + if let Err(e) = store.get_bucket_info(&input.bucket, &BucketOptions {}).await { if DiskError::VolumeNotFound.is(&e) { return Err(s3_error!(NoSuchBucket)); } else { @@ -243,11 +275,14 @@ impl S3 for FS { let h = HeaderMap::new(); let opts = &ObjectOptions::default(); - let reader = try_!( - self.store - .get_object_reader(bucket.as_str(), key.as_str(), range, h, opts) - .await - ); + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + let reader = try_!(store.get_object_reader(bucket.as_str(), key.as_str(), range, h, opts).await); let info = reader.object_info; @@ -270,7 +305,14 @@ impl S3 for FS { async fn head_bucket(&self, req: S3Request) -> S3Result> { let input = req.input; - if let Err(e) = self.store.get_bucket_info(&input.bucket, &BucketOptions {}).await { + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + if let Err(e) = store.get_bucket_info(&input.bucket, &BucketOptions {}).await { if DiskError::VolumeNotFound.is(&e) { return Err(s3_error!(NoSuchBucket)); } else { @@ -287,7 +329,14 @@ impl S3 for FS { // mc get 2 let HeadObjectInput { bucket, key, .. } = req.input; - let info = try_!(self.store.get_object_info(&bucket, &key, &ObjectOptions::default()).await); + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + let info = try_!(store.get_object_info(&bucket, &key, &ObjectOptions::default()).await); debug!("info {:?}", info); let content_type = try_!(ContentType::from_str("application/x-msdownload")); @@ -307,7 +356,14 @@ impl S3 for FS { async fn list_buckets(&self, _: S3Request) -> S3Result> { // mc ls - let bucket_infos = try_!(self.store.list_bucket(&BucketOptions {}).await); + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + let bucket_infos = try_!(store.list_bucket(&BucketOptions {}).await); let buckets: Vec = bucket_infos .iter() @@ -355,8 +411,15 @@ impl S3 for FS { let prefix = prefix.unwrap_or_default(); let delimiter = delimiter.unwrap_or_default(); + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + let object_infos = try_!( - self.store + store .list_objects_v2( &bucket, &prefix, @@ -435,9 +498,16 @@ impl S3 for FS { let reader = PutObjReader::new(body, content_length as usize); - try_!(self.store.put_object(&bucket, &key, reader, &ObjectOptions::default()).await); + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; - // self.store.put_object(bucket, object, data, opts); + try_!(store.put_object(&bucket, &key, reader, &ObjectOptions::default()).await); + + // store.put_object(bucket, object, data, opts); let output = PutObjectOutput { ..Default::default() }; Ok(S3Response::new(output)) @@ -456,11 +526,15 @@ impl S3 for FS { debug!("create_multipart_upload meta {:?}", &metadata); - let MultipartUploadResult { upload_id, .. } = try_!( - self.store - .new_multipart_upload(&bucket, &key, &ObjectOptions::default()) - .await - ); + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + let MultipartUploadResult { upload_id, .. } = + try_!(store.new_multipart_upload(&bucket, &key, &ObjectOptions::default()).await); let output = CreateMultipartUploadOutput { bucket: Some(bucket), @@ -495,11 +569,14 @@ impl S3 for FS { let data = PutObjReader::new(body, content_length as usize); let opts = ObjectOptions::default(); - try_!( - self.store - .put_object_part(&bucket, &key, &upload_id, part_id, data, &opts) - .await - ); + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + + try_!(store.put_object_part(&bucket, &key, &upload_id, part_id, data, &opts).await); let output = UploadPartOutput { ..Default::default() }; Ok(S3Response::new(output)) @@ -554,8 +631,15 @@ impl S3 for FS { uploaded_parts.push(CompletePart::from(part)); } + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + try_!( - self.store + store .complete_multipart_upload(&bucket, &key, &upload_id, uploaded_parts, opts) .await ); @@ -577,9 +661,16 @@ impl S3 for FS { bucket, key, upload_id, .. } = req.input; + let layer = new_object_layer_fn(); + let lock = layer.lock().await; + let store = match lock.as_ref() { + Some(s) => s, + None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), + }; + let opts = &ObjectOptions::default(); try_!( - self.store + store .abort_multipart_upload(bucket.as_str(), key.as_str(), upload_id.as_str(), opts) .await ); From fc533c7b2b79333b708d3464bd9511e1b360db64 Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 10 Sep 2024 17:32:39 +0800 Subject: [PATCH 13/23] test global_disks --- rustfs/src/main.rs | 140 +++++++++++++++++++------------------ rustfs/src/storage/ecfs.rs | 4 +- 2 files changed, 75 insertions(+), 69 deletions(-) diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 94cca1cf..0f9fbcde 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -69,84 +69,90 @@ async fn run(opt: config::Opt) -> Result<()> { // 用于rpc let (endpoint_pools, _) = EndpointServerPools::from_volumes(opt.address.clone().as_str(), opt.volumes.clone())?; - // Setup S3 service - // 本项目使用s3s库来实现s3服务 - let service = { - // let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), endpoint_pools).await?); - let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new().await?); - //设置AK和SK - //其中部份内容从config配置文件中读取 - let mut access_key = String::from_str(config::DEFAULT_ACCESS_KEY).unwrap(); - let mut secret_key = String::from_str(config::DEFAULT_SECRET_KEY).unwrap(); + tokio::spawn(async move { + // Setup S3 service + // 本项目使用s3s库来实现s3服务 + let service = { + // let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(opt.address.clone(), endpoint_pools).await?); + let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new()); + //设置AK和SK + //其中部份内容从config配置文件中读取 + let mut access_key = String::from_str(config::DEFAULT_ACCESS_KEY).unwrap(); + let mut secret_key = String::from_str(config::DEFAULT_SECRET_KEY).unwrap(); - // Enable authentication - if let (Some(ak), Some(sk)) = (opt.access_key, opt.secret_key) { - access_key = ak; - secret_key = sk; - } - //显示info信息 - info!("authentication is enabled {}, {}", &access_key, &secret_key); - b.set_auth(SimpleAuth::from_single(access_key, secret_key)); - - // Enable parsing virtual-hosted-style requests - if let Some(dm) = opt.domain_name { - info!("virtual-hosted-style requests are enabled use domain_name {}", &dm); - b.set_base_domain(dm); - } - - // if domain_name.is_some() { - // info!( - // "virtual-hosted-style requests are enabled use domain_name {}", - // domain_name.as_ref().unwrap() - // ); - // b.set_base_domain(domain_name.unwrap()); - // } - - b.build() - }; - - let hyper_service = service.into_shared(); - - let http_server = ConnBuilder::new(TokioExecutor::new()); - let graceful = hyper_util::server::graceful::GracefulShutdown::new(); - - let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c()); - - info!("server is running at http://{local_addr}"); - - loop { - let (socket, _) = tokio::select! { - res = listener.accept() => { - match res { - Ok(conn) => conn, - Err(err) => { - tracing::error!("error accepting connection: {err}"); - continue; - } - } + // Enable authentication + if let (Some(ak), Some(sk)) = (opt.access_key, opt.secret_key) { + access_key = ak; + secret_key = sk; } - _ = ctrl_c.as_mut() => { - break; + //显示info信息 + info!("authentication is enabled {}, {}", &access_key, &secret_key); + b.set_auth(SimpleAuth::from_single(access_key, secret_key)); + + // Enable parsing virtual-hosted-style requests + if let Some(dm) = opt.domain_name { + info!("virtual-hosted-style requests are enabled use domain_name {}", &dm); + b.set_base_domain(dm); } + + // if domain_name.is_some() { + // info!( + // "virtual-hosted-style requests are enabled use domain_name {}", + // domain_name.as_ref().unwrap() + // ); + // b.set_base_domain(domain_name.unwrap()); + // } + + b.build() }; - let conn = http_server.serve_connection(TokioIo::new(socket), hyper_service.clone()); - let conn = graceful.watch(conn.into_owned()); - tokio::spawn(async move { - let _ = conn.await; - }); - } + let hyper_service = service.into_shared(); + + let http_server = ConnBuilder::new(TokioExecutor::new()); + let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c()); + let graceful = hyper_util::server::graceful::GracefulShutdown::new(); + info!("server is running at http://{local_addr}"); + + loop { + let (socket, _) = tokio::select! { + res = listener.accept() => { + match res { + Ok(conn) => conn, + Err(err) => { + tracing::error!("error accepting connection: {err}"); + continue; + } + } + } + _ = ctrl_c.as_mut() => { + break; + } + }; + + let conn = http_server.serve_connection(TokioIo::new(socket), hyper_service.clone()); + let conn = graceful.watch(conn.into_owned()); + tokio::spawn(async move { + let _ = conn.await; + }); + } + + tokio::select! { + () = graceful.shutdown() => { + tracing::debug!("Gracefully shutdown!"); + }, + () = tokio::time::sleep(std::time::Duration::from_secs(10)) => { + tracing::debug!("Waited 10 seconds for graceful shutdown, aborting..."); + } + } + }); warn!(" init store"); // init store ECStore::new(opt.address.clone(), endpoint_pools.clone()).await?; tokio::select! { - () = graceful.shutdown() => { - tracing::debug!("Gracefully shutdown!"); - }, - () = tokio::time::sleep(std::time::Duration::from_secs(10)) => { - tracing::debug!("Waited 10 seconds for graceful shutdown, aborting..."); + _ = tokio::signal::ctrl_c() => { + } } diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index f7a50bae..bfb9edd9 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -45,9 +45,9 @@ pub struct FS { } impl FS { - pub async fn new() -> Result { + pub fn new() -> Self { // let store: ECStore = ECStore::new(address, endpoint_pools).await?; - Ok(Self {}) + Self {} } } #[async_trait::async_trait] From c494bd56edca2da151d2c1cfdbc02a51f664ddcd Mon Sep 17 00:00:00 2001 From: weisd Date: Tue, 10 Sep 2024 18:06:15 +0800 Subject: [PATCH 14/23] init global_local_disks --- ecstore/src/store.rs | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index b0d42bbe..38d74b34 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -28,6 +28,7 @@ use lazy_static::lazy_static; lazy_static! { pub static ref GLOBAL_OBJECT_API: Arc>> = Arc::new(Mutex::new(None)); + pub static ref GLOBAL_LOCAL_DISK: Arc>>> = Arc::new(Mutex::new(Vec::new())); } pub fn new_object_layer_fn() -> Arc>> { @@ -129,9 +130,11 @@ impl ECStore { Ok(()) } - pub fn local_disks(&self) -> Vec { - self.local_disks.clone() - } + pub fn init_local_disks() {} + + // pub fn local_disks(&self) -> Vec { + // self.local_disks.clone() + // } fn single_pool(&self) -> bool { self.pools.len() == 1 From 7e108f167d1871bc7bf0234e04cba625482043ef Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 11 Sep 2024 14:25:53 +0800 Subject: [PATCH 15/23] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=85=A8=E5=B1=80disk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecstore/src/disk/local.rs | 61 ++++++++++++----- ecstore/src/disk/mod.rs | 4 +- ecstore/src/disk/remote.rs | 19 ++++-- ecstore/src/peer.rs | 43 ++++++------ ecstore/src/sets.rs | 30 +++++++-- ecstore/src/store.rs | 134 +++++++++++++++++++++++++++++++++---- ecstore/src/store_init.rs | 28 +++++--- rustfs/src/grpc.rs | 38 +++++------ rustfs/src/main.rs | 17 ++++- rustfs/src/storage/ecfs.rs | 30 ++++----- 10 files changed, 301 insertions(+), 103 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index a58d5fe7..50e71e5f 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -19,18 +19,29 @@ use std::{ use time::OffsetDateTime; use tokio::fs::{self, File}; use tokio::io::ErrorKind; +use tokio::sync::Mutex; use tracing::{debug, warn}; use uuid::Uuid; +#[derive(Debug)] +pub struct FormatInfo { + pub id: Option, + pub data: Vec, + pub file_info: Option, + pub last_check: Option, +} + +impl FormatInfo {} + #[derive(Debug)] pub struct LocalDisk { pub root: PathBuf, - pub id: Uuid, - pub _format_data: Vec, - pub _format_meta: Option, pub _format_path: PathBuf, - // pub format_legacy: bool, // drop - pub _format_last_check: Option, + pub format_info: Mutex, + // pub id: Mutex>, + // pub format_data: Mutex>, + // pub format_file_info: Mutex>, + // pub format_last_check: Mutex>, } impl LocalDisk { @@ -48,7 +59,7 @@ impl LocalDisk { let (format_data, format_meta) = read_file_exists(&format_path).await?; - let mut id = Uuid::nil(); + let mut id = None; // let mut format_legacy = false; let mut format_last_check = None; @@ -61,19 +72,26 @@ impl LocalDisk { return Err(Error::from(DiskError::InconsistentDisk)); } - id = fm.erasure.this; + id = Some(fm.erasure.this); // format_legacy = fm.erasure.distribution_algo == DistributionAlgoVersion::V1; format_last_check = Some(OffsetDateTime::now_utc()); } + let format_info = FormatInfo { + id, + data: format_data, + file_info: format_meta, + last_check: format_last_check, + }; + let disk = Self { root, - id, - _format_meta: format_meta, - _format_data: format_data, _format_path: format_path, - // format_legacy, - _format_last_check: format_last_check, + format_info: Mutex::new(format_info), + // // format_legacy, + // format_file_info: Mutex::new(format_meta), + // format_data: Mutex::new(format_data), + // format_last_check: Mutex::new(format_last_check), }; disk.make_meta_volumes().await?; @@ -389,15 +407,26 @@ impl DiskAPI for LocalDisk { fn is_local(&self) -> bool { true } - - fn id(&self) -> Uuid { - self.id + async fn close(&self) -> Result<()> { + Ok(()) } - fn path(&self) -> PathBuf { self.root.clone() } + async fn get_disk_id(&self) -> Option { + // TODO: check format file + let format_info = self.format_info.lock().await; + + format_info.id.clone() + // TODO: 判断源文件id,是否有效 + } + + async fn set_disk_id(&self, _id: Option) -> Result<()> { + // 本地不需要设置 + Ok(()) + } + #[must_use] async fn read_all(&self, volume: &str, path: &str) -> Result { let p = self.get_object_path(volume, path)?; diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 1256dbc3..19b8a104 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -46,8 +46,10 @@ pub async fn new_disk(ep: &endpoint::Endpoint, opt: &DiskOption) -> Result bool; - fn id(&self) -> Uuid; fn path(&self) -> PathBuf; + async fn close(&self) -> Result<()>; + async fn get_disk_id(&self) -> Option; + async fn set_disk_id(&self, id: Option) -> Result<()>; async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()>; async fn read_all(&self, volume: &str, path: &str) -> Result; diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 68398ac8..2024d04b 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -1,6 +1,7 @@ use std::{path::PathBuf, sync::RwLock, time::Duration}; use bytes::Bytes; +use futures::lock::Mutex; use protos::{ node_service_time_out_client, proto_gen::node_service::{ @@ -32,6 +33,7 @@ use super::{ #[derive(Debug)] pub struct RemoteDisk { + id: Mutex>, channel: RwLock>, url: url::Url, pub root: PathBuf, @@ -45,6 +47,7 @@ impl RemoteDisk { channel: RwLock::new(None), url: ep.url.clone(), root, + id: Mutex::new(None), }) } @@ -83,15 +86,23 @@ impl DiskAPI for RemoteDisk { fn is_local(&self) -> bool { false } - - fn id(&self) -> Uuid { - Uuid::nil() + async fn close(&self) -> Result<()> { + Ok(()) } - fn path(&self) -> PathBuf { self.root.clone() } + async fn get_disk_id(&self) -> Option { + self.id.lock().await.clone() + } + async fn set_disk_id(&self, id: Option) -> Result<()> { + let mut lock = self.id.lock().await; + *lock = id; + + Ok(()) + } + async fn read_all(&self, volume: &str, path: &str) -> Result { let mut client = self.get_client(); let request = Request::new(ReadAllRequest { diff --git a/ecstore/src/peer.rs b/ecstore/src/peer.rs index 8d80c6dc..e2c8f163 100644 --- a/ecstore/src/peer.rs +++ b/ecstore/src/peer.rs @@ -8,8 +8,9 @@ use tonic::transport::{Channel, Endpoint}; use tonic::Request; use tracing::warn; +use crate::store::all_local_disk; use crate::{ - disk::{self, error::DiskError, DiskStore, VolumeInfo}, + disk::{self, error::DiskError, VolumeInfo}, endpoints::{EndpointServerPools, Node}, error::{Error, Result}, store_api::{BucketInfo, BucketOptions, MakeBucketOptions}, @@ -33,21 +34,20 @@ pub struct S3PeerSys { } impl S3PeerSys { - pub fn new(eps: &EndpointServerPools, local_disks: Vec) -> Self { + pub fn new(eps: &EndpointServerPools) -> Self { Self { - clients: Self::new_clients(eps, local_disks), + clients: Self::new_clients(eps), pools_count: eps.as_ref().len(), } } - fn new_clients(eps: &EndpointServerPools, local_disks: Vec) -> Vec { + fn new_clients(eps: &EndpointServerPools) -> Vec { let nodes = eps.get_nodes(); let v: Vec = nodes .iter() .map(|e| { if e.is_local { - let cli: Box = - Box::new(LocalPeerS3Client::new(local_disks.clone(), Some(e.clone()), Some(e.pools.clone()))); + let cli: Box = Box::new(LocalPeerS3Client::new(Some(e.clone()), Some(e.pools.clone()))); Arc::new(cli) } else { let cli: Box = Box::new(RemotePeerS3Client::new(Some(e.clone()), Some(e.pools.clone()))); @@ -216,15 +216,15 @@ impl S3PeerSys { #[derive(Debug)] pub struct LocalPeerS3Client { - pub local_disks: Vec, + // pub local_disks: Vec, // pub node: Node, pub pools: Option>, } impl LocalPeerS3Client { - pub fn new(local_disks: Vec, _node: Option, pools: Option>) -> Self { + pub fn new(_node: Option, pools: Option>) -> Self { Self { - local_disks, + // local_disks, // node, pools, } @@ -237,8 +237,10 @@ impl PeerS3Client for LocalPeerS3Client { self.pools.clone() } async fn list_bucket(&self, _opts: &BucketOptions) -> Result> { - let mut futures = Vec::with_capacity(self.local_disks.len()); - for disk in self.local_disks.iter() { + let local_disks = all_local_disk().await; + + let mut futures = Vec::with_capacity(local_disks.len()); + for disk in local_disks.iter() { futures.push(disk.list_volumes()); } @@ -283,8 +285,9 @@ impl PeerS3Client for LocalPeerS3Client { Ok(buckets) } async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { - let mut futures = Vec::with_capacity(self.local_disks.len()); - for disk in self.local_disks.iter() { + let local_disks = all_local_disk().await; + let mut futures = Vec::with_capacity(local_disks.len()); + for disk in local_disks.iter() { futures.push(async move { match disk.make_volume(bucket).await { Ok(_) => Ok(()), @@ -316,15 +319,16 @@ impl PeerS3Client for LocalPeerS3Client { } async fn get_bucket_info(&self, bucket: &str, _opts: &BucketOptions) -> Result { - let mut futures = Vec::with_capacity(self.local_disks.len()); - for disk in self.local_disks.iter() { + let local_disks = all_local_disk().await; + let mut futures = Vec::with_capacity(local_disks.len()); + for disk in local_disks.iter() { futures.push(disk.stat_volume(bucket)); } let results = join_all(futures).await; - let mut ress = Vec::with_capacity(self.local_disks.len()); - let mut errs = Vec::with_capacity(self.local_disks.len()); + let mut ress = Vec::with_capacity(local_disks.len()); + let mut errs = Vec::with_capacity(local_disks.len()); for res in results { match res { @@ -354,9 +358,10 @@ impl PeerS3Client for LocalPeerS3Client { } async fn delete_bucket(&self, bucket: &str) -> Result<()> { - let mut futures = Vec::with_capacity(self.local_disks.len()); + let local_disks = all_local_disk().await; + let mut futures = Vec::with_capacity(local_disks.len()); - for disk in self.local_disks.iter() { + for disk in local_disks.iter() { futures.push(disk.delete_volume(bucket)); } diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index 63815089..4d363595 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -12,6 +12,7 @@ use crate::{ endpoints::PoolEndpoints, error::{Error, Result}, set_disk::SetDisks, + store::{GLOBAL_IsDistErasure, GLOBAL_LOCAL_DISK_SET_DRIVES}, store_api::{ BucketInfo, BucketOptions, CompletePart, DeletedObject, GetObjectReader, HTTPRangeSpec, ListObjectsV2Info, MakeBucketOptions, MultipartUploadResult, ObjectInfo, ObjectOptions, ObjectToDelete, PartInfo, PutObjReader, StorageAPI, @@ -35,7 +36,7 @@ pub struct Sets { } impl Sets { - pub fn new( + pub async fn new( disks: Vec>, endpoints: &PoolEndpoints, fm: &FormatV3, @@ -51,11 +52,32 @@ impl Sets { let mut set_drive = Vec::with_capacity(set_drive_count); for j in 0..set_drive_count { let idx = i * set_drive_count + j; - if disks[idx].is_none() { + let mut disk = disks[idx].clone(); + if disk.is_none() { set_drive.push(None); - } else { - let disk = disks[idx].clone(); + continue; + } + + if disk.as_ref().unwrap().is_local() && *GLOBAL_IsDistErasure.read().await { + let local_disk = { + let local_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.read().await; + local_set_drives[pool_idx][i][j].clone() + }; + + if local_disk.is_none() { + set_drive.push(None); + continue; + } + + let _ = disk.as_ref().unwrap().close().await; + + disk = local_disk; + } + + if let Some(_disk_id) = disk.as_ref().unwrap().get_disk_id().await { set_drive.push(disk); + } else { + set_drive.push(None); } } diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 38d74b34..a0303874 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -1,7 +1,9 @@ use crate::{ bucket_meta::BucketMetadata, - disk::{error::DiskError, DeleteOptions, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET}, - endpoints::EndpointServerPools, + disk::{ + error::DiskError, new_disk, DeleteOptions, DiskOption, DiskStore, WalkDirOptions, BUCKET_META_PREFIX, RUSTFS_META_BUCKET, + }, + endpoints::{EndpointServerPools, SetupType}, error::{Error, Result}, peer::S3PeerSys, sets::Sets, @@ -20,24 +22,124 @@ use std::{ sync::Arc, }; use time::OffsetDateTime; -use tokio::sync::Mutex; +use tokio::{fs, sync::RwLock}; use tracing::{debug, warn}; use uuid::Uuid; use lazy_static::lazy_static; lazy_static! { - pub static ref GLOBAL_OBJECT_API: Arc>> = Arc::new(Mutex::new(None)); - pub static ref GLOBAL_LOCAL_DISK: Arc>>> = Arc::new(Mutex::new(Vec::new())); + pub static ref GLOBAL_IsErasure: RwLock = RwLock::new(false); + pub static ref GLOBAL_IsDistErasure: RwLock = RwLock::new(false); + pub static ref GLOBAL_IsErasureSD: RwLock = RwLock::new(false); } -pub fn new_object_layer_fn() -> Arc>> { - // 这里不需要显式地锁定和解锁，因为 Arc 提供了必要的线程安全性 +pub async fn update_erasure_type(setup_type: SetupType) { + let mut is_erasure = GLOBAL_IsErasure.write().await; + *is_erasure = setup_type == SetupType::Erasure; + + let mut is_dist_erasure = GLOBAL_IsDistErasure.write().await; + *is_dist_erasure = setup_type == SetupType::DistErasure; + + if *is_dist_erasure { + *is_erasure = true + } + + let mut is_erasure_sd = GLOBAL_IsErasureSD.write().await; + *is_erasure_sd = setup_type == SetupType::ErasureSD; +} + +lazy_static! { + pub static ref GLOBAL_LOCAL_DISK_MAP: Arc>>> = Arc::new(RwLock::new(HashMap::new())); + pub static ref GLOBAL_LOCAL_DISK_SET_DRIVES: Arc>>>>> = + Arc::new(RwLock::new(Vec::new())); +} + +pub async fn find_local_disk(disk_path: &String) -> Option { + let disk_path = match fs::canonicalize(disk_path).await { + Ok(disk_path) => disk_path, + Err(_) => return None, + }; + + let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await; + + let path = disk_path.to_string_lossy().to_string(); + if disk_map.contains_key(&path) { + let a = disk_map[&path].as_ref().cloned(); + + return a; + } + None +} + +pub async fn all_local_disk_path() -> Vec { + let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await; + disk_map.keys().map(|v| v.clone()).collect() +} + +pub async fn all_local_disk() -> Vec { + let disk_map = GLOBAL_LOCAL_DISK_MAP.read().await; + disk_map + .values() + .filter(|v| v.is_some()) + .map(|v| v.as_ref().unwrap().clone()) + .collect() +} + +// init_local_disks 初始化本地磁盘，server启动前必须初始化成功 +pub async fn init_local_disks(endpoint_pools: EndpointServerPools) -> Result<()> { + let opt = &DiskOption { + cleanup: true, + health_check: true, + }; + + let mut global_set_drives = GLOBAL_LOCAL_DISK_SET_DRIVES.write().await; + for pool_eps in endpoint_pools.as_ref().iter() { + let mut set_count_drives = Vec::with_capacity(pool_eps.set_count); + for _ in 0..pool_eps.set_count { + set_count_drives.push(vec![None; pool_eps.drives_per_set]); + } + + global_set_drives.push(set_count_drives); + } + + let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await; + + for pool_eps in endpoint_pools.as_ref().iter() { + let mut set_drives = HashMap::new(); + for ep in pool_eps.endpoints.as_ref().iter() { + if !ep.is_local { + continue; + } + + let disk = new_disk(ep, opt).await?; + + let path = disk.path().to_string_lossy().to_string(); + + global_local_disk_map.insert(path, Some(disk.clone())); + + set_drives.insert(ep.disk_idx, Some(disk.clone())); + + if ep.pool_idx.is_some() && ep.set_idx.is_some() && ep.disk_idx.is_some() { + global_set_drives[ep.pool_idx.unwrap()][ep.set_idx.unwrap()][ep.disk_idx.unwrap()] = Some(disk.clone()); + } + } + } + + Ok(()) +} + +lazy_static! { + pub static ref GLOBAL_OBJECT_API: Arc>> = Arc::new(RwLock::new(None)); + pub static ref GLOBAL_LOCAL_DISK: Arc>>> = Arc::new(RwLock::new(Vec::new())); +} + +pub fn new_object_layer_fn() -> Arc>> { GLOBAL_OBJECT_API.clone() } async fn set_object_layer(o: ECStore) { - let mut global_object_api = GLOBAL_OBJECT_API.lock().await; + let mut global_object_api = GLOBAL_OBJECT_API.write().await; *global_object_api = Some(o); } @@ -48,7 +150,7 @@ pub struct ECStore { pub disk_map: HashMap>>, pub pools: Vec, pub peer_sys: S3PeerSys, - pub local_disks: Vec, + // pub local_disks: Vec, } impl ECStore { @@ -108,20 +210,28 @@ impl ECStore { } } - let sets = Sets::new(disks.clone(), pool_eps, &fm, i, partiy_count)?; + let sets = Sets::new(disks.clone(), pool_eps, &fm, i, partiy_count).await?; pools.push(sets); disk_map.insert(i, disks); } - let peer_sys = S3PeerSys::new(&endpoint_pools, local_disks.clone()); + // 替换本地磁盘 + if !*GLOBAL_IsDistErasure.read().await { + let mut global_local_disk_map = GLOBAL_LOCAL_DISK_MAP.write().await; + for disk in local_disks { + let path = disk.path().to_string_lossy().to_string(); + global_local_disk_map.insert(path, Some(disk.clone())); + } + } + + let peer_sys = S3PeerSys::new(&endpoint_pools); let ec = ECStore { id: deployment_id.unwrap(), disk_map, pools, - local_disks, peer_sys, }; diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index a1178d88..e18ebe13 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -1,7 +1,9 @@ use crate::{ - disk::error::DiskError, - disk::format::{FormatErasureVersion, FormatMetaVersion, FormatV3}, - disk::{new_disk, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET}, + disk::{ + error::DiskError, + format::{FormatErasureVersion, FormatMetaVersion, FormatV3}, + new_disk, DiskOption, DiskStore, FORMAT_CONFIG_FILE, RUSTFS_META_BUCKET, + }, endpoints::Endpoints, error::{Error, Result}, }; @@ -10,6 +12,7 @@ use std::{ collections::{hash_map::Entry, HashMap}, fmt::Debug, }; + use tracing::warn; use uuid::Uuid; @@ -188,17 +191,22 @@ pub fn default_partiy_count(drive: usize) -> usize { async fn read_format_file_all(disks: &[Option], heal: bool) -> (Vec>, Vec>) { let mut futures = Vec::with_capacity(disks.len()); - for ep in disks.iter() { - futures.push(read_format_file(ep, heal)); + for disk in disks.iter() { + futures.push(read_format_file(disk, heal)); } let mut datas = Vec::with_capacity(disks.len()); let mut errors = Vec::with_capacity(disks.len()); let results = join_all(futures).await; + let mut i = 0; for result in results { match result { Ok(s) => { + if !heal { + let _ = disks[i].as_ref().unwrap().set_disk_id(Some(s.erasure.this.clone())).await; + } + datas.push(Some(s)); errors.push(None); } @@ -207,6 +215,8 @@ async fn read_format_file_all(disks: &[Option], heal: bool) -> (Vec, _heal: bool) -> Result], formats: &[Option]) -> Vec> { let mut futures = Vec::with_capacity(disks.len()); - for (i, ep) in disks.iter().enumerate() { - futures.push(save_format_file(ep, &formats[i])); + for (i, disk) in disks.iter().enumerate() { + futures.push(save_format_file(disk, &formats[i])); } let mut errors = Vec::with_capacity(disks.len()); @@ -277,9 +287,7 @@ async fn save_format_file(disk: &Option, format: &Option) - disk.rename_file(RUSTFS_META_BUCKET, tmpfile.as_str(), RUSTFS_META_BUCKET, FORMAT_CONFIG_FILE) .await?; - // let mut disk = disk; - - // disk.set_disk_id(format.erasure.this); + disk.set_disk_id(Some(format.erasure.this)).await?; Ok(()) } diff --git a/rustfs/src/grpc.rs b/rustfs/src/grpc.rs index b9edd6a1..cf6e69be 100644 --- a/rustfs/src/grpc.rs +++ b/rustfs/src/grpc.rs @@ -1,11 +1,10 @@ use ecstore::{ disk::{DeleteOptions, DiskStore, ReadMultipleReq, ReadOptions, WalkDirOptions}, - endpoints::EndpointServerPools, erasure::{ReadAt, Write}, peer::{LocalPeerS3Client, PeerS3Client}, + store::{all_local_disk_path, find_local_disk}, store_api::{BucketOptions, FileInfo, MakeBucketOptions}, }; -use tokio::fs; use tonic::{Request, Response, Status}; use tracing::{debug, error, info}; @@ -29,28 +28,29 @@ struct NodeService { pub local_peer: LocalPeerS3Client, } -pub fn make_server(endpoint_pools: EndpointServerPools) -> NodeServer { - // TODO: 参考rustfs创建 https://github.com/rustfs/s3-rustfs/issues/30#issuecomment-2339664516 - let local_disks = Vec::new(); - let local_peer = LocalPeerS3Client::new(local_disks, None, None); +pub fn make_server() -> NodeServer { + // let local_disks = all_local_disk().await; + let local_peer = LocalPeerS3Client::new(None, None); NodeServer::new(NodeService { local_peer }) } impl NodeService { async fn find_disk(&self, disk_path: &String) -> Option { - let disk_path = match fs::canonicalize(disk_path).await { - Ok(disk_path) => disk_path, - Err(_) => return None, - }; - self.local_peer.local_disks.iter().find(|&x| x.path() == disk_path).cloned() + find_local_disk(disk_path).await + // let disk_path = match fs::canonicalize(disk_path).await { + // Ok(disk_path) => disk_path, + // Err(_) => return None, + // }; + // self.local_peer.local_disks.iter().find(|&x| x.path() == disk_path).cloned() } - fn all_disk(&self) -> Vec { - self.local_peer - .local_disks - .iter() - .map(|disk| disk.path().to_string_lossy().to_string()) - .collect() + async fn all_disk(&self) -> Vec { + all_local_disk_path().await + // self.local_peer + // .local_disks + // .iter() + // .map(|disk| disk.path().to_string_lossy().to_string()) + // .collect() } } @@ -501,7 +501,7 @@ impl Node for NodeService { } else { Ok(tonic::Response::new(MakeVolumesResponse { success: false, - error_info: Some(format!("can not find disk, all disks: {:?}", self.all_disk())), + error_info: Some(format!("can not find disk, all disks: {:?}", self.all_disk().await)), })) } } @@ -522,7 +522,7 @@ impl Node for NodeService { } else { Ok(tonic::Response::new(MakeVolumeResponse { success: false, - error_info: Some(format!("can not find disk, all disks: {:?}", self.all_disk())), + error_info: Some(format!("can not find disk, all disks: {:?}", self.all_disk().await)), })) } } diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index 827df794..a3e962f5 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -4,7 +4,11 @@ mod service; mod storage; use clap::Parser; -use ecstore::{endpoints::EndpointServerPools, error::Result, store::ECStore}; +use ecstore::{ + endpoints::EndpointServerPools, + error::Result, + store::{init_local_disks, update_erasure_type, ECStore}, +}; use grpc::make_server; use hyper_util::{ rt::{TokioExecutor, TokioIo}, @@ -72,7 +76,13 @@ async fn run(opt: config::Opt) -> Result<()> { // }; // 用于rpc - let (endpoint_pools, _) = EndpointServerPools::from_volumes(opt.address.clone().as_str(), opt.volumes.clone())?; + let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(opt.address.clone().as_str(), opt.volumes.clone())?; + + update_erasure_type(setup_type).await; + + // 初始化本地磁盘 + init_local_disks(endpoint_pools.clone()).await?; + // Setup S3 service // 本项目使用s3s库来实现s3服务 let service = { @@ -108,7 +118,8 @@ async fn run(opt: config::Opt) -> Result<()> { b.build() }; - let rpc_service = make_server(endpoint_pools.clone()); + + let rpc_service = make_server(); tokio::spawn(async move { let hyper_service = service.into_shared(); diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index bfb9edd9..6a03c51a 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -61,7 +61,7 @@ impl S3 for FS { let input = req.input; let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -94,7 +94,7 @@ impl S3 for FS { let input = req.input; let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -125,7 +125,7 @@ impl S3 for FS { let objects: Vec = vec![dobj]; let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -192,7 +192,7 @@ impl S3 for FS { .collect(); let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -233,7 +233,7 @@ impl S3 for FS { let input = req.input; let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -276,7 +276,7 @@ impl S3 for FS { let opts = &ObjectOptions::default(); let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -306,7 +306,7 @@ impl S3 for FS { let input = req.input; let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -330,7 +330,7 @@ impl S3 for FS { let HeadObjectInput { bucket, key, .. } = req.input; let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -357,7 +357,7 @@ impl S3 for FS { // mc ls let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -412,7 +412,7 @@ impl S3 for FS { let delimiter = delimiter.unwrap_or_default(); let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -499,7 +499,7 @@ impl S3 for FS { let reader = PutObjReader::new(body, content_length as usize); let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -527,7 +527,7 @@ impl S3 for FS { debug!("create_multipart_upload meta {:?}", &metadata); let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -570,7 +570,7 @@ impl S3 for FS { let opts = ObjectOptions::default(); let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -632,7 +632,7 @@ impl S3 for FS { } let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), @@ -662,7 +662,7 @@ impl S3 for FS { } = req.input; let layer = new_object_layer_fn(); - let lock = layer.lock().await; + let lock = layer.read().await; let store = match lock.as_ref() { Some(s) => s, None => return Err(S3Error::with_message(S3ErrorCode::InternalError, format!("Not init",))), From 1aa9af80818d9bd4918f031fd527c47d8cd8665c Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 11 Sep 2024 14:27:02 +0800 Subject: [PATCH 16/23] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=85=A8=E5=B1=80disk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecstore/src/disk/local.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 50e71e5f..5cab96a8 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -26,9 +26,9 @@ use uuid::Uuid; #[derive(Debug)] pub struct FormatInfo { pub id: Option, - pub data: Vec, - pub file_info: Option, - pub last_check: Option, + pub _data: Vec, + pub _file_info: Option, + pub _last_check: Option, } impl FormatInfo {} From 35f0d7a82a0304443cfbe6e47cff0ab0dc216330 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 11 Sep 2024 14:34:03 +0800 Subject: [PATCH 17/23] =?UTF-8?q?=E6=9B=B4=E6=96=B0=E5=85=A8=E5=B1=80disk?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- ecstore/src/disk/local.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 5cab96a8..50e71e5f 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -26,9 +26,9 @@ use uuid::Uuid; #[derive(Debug)] pub struct FormatInfo { pub id: Option, - pub _data: Vec, - pub _file_info: Option, - pub _last_check: Option, + pub data: Vec, + pub file_info: Option, + pub last_check: Option, } impl FormatInfo {} From aa8135a1adfc83849cb1befe41c9cfb869f26cd3 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 11 Sep 2024 15:19:59 +0800 Subject: [PATCH 18/23] test get_format_file_in_quorum --- ecstore/src/store_init.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index e18ebe13..1a09e6ee 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -127,8 +127,15 @@ fn get_format_file_in_quorum(formats: &[Option]) -> Result { let (max_drives, max_count) = countmap.iter().max_by_key(|&(_, c)| c).unwrap_or((&0, &0)); + warn!("get_format_file_in_quorum fi: {:?}", &formats); + if *max_drives == 0 || *max_count < formats.len() / 2 { - warn!("*max_drives == 0 || *max_count < formats.len() / 2"); + warn!( + "*max_drives == 0 || *max_count < formats.len() / 2, {} || {}<{}", + max_drives, + max_count, + formats.len() / 2 + ); return Err(Error::new(ErasureError::ErasureReadQuorum)); } From 4eec956905a21678624a10a2b7f281c5f98c5343 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 11 Sep 2024 15:26:54 +0800 Subject: [PATCH 19/23] test get_format_file_in_quorum --- ecstore/src/store_init.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index 1a09e6ee..6f28c0dd 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -50,10 +50,14 @@ pub async fn do_init_format_file( set_drive_count: usize, deployment_id: Option, ) -> Result { + warn!("do_init_format_file id: {:?}, first_disk: {}", deployment_id, first_disk); + let (formats, errs) = read_format_file_all(disks, false).await; DiskError::check_disk_fatal_errs(&errs)?; + warn!("read_format_file_all errs {:?}", &errs); + check_format_erasure_values(&formats, set_drive_count)?; if first_disk && DiskError::should_init_erasure_disks(&errs) { From f593cce6c90c5b708a9ed807ada7be8917c84554 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 11 Sep 2024 16:26:04 +0800 Subject: [PATCH 20/23] update fn name --- ecstore/src/store_init.rs | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index 6f28c0dd..245f0b27 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -43,34 +43,34 @@ pub async fn init_disks(eps: &Endpoints, opt: &DiskOption) -> (Vec], set_count: usize, set_drive_count: usize, deployment_id: Option, ) -> Result { - warn!("do_init_format_file id: {:?}, first_disk: {}", deployment_id, first_disk); + warn!("connect_load_init_formats id: {:?}, first_disk: {}", deployment_id, first_disk); - let (formats, errs) = read_format_file_all(disks, false).await; + let (formats, errs) = load_format_erasure_all(disks, false).await; DiskError::check_disk_fatal_errs(&errs)?; - warn!("read_format_file_all errs {:?}", &errs); + warn!("load_format_erasure_all errs {:?}", &errs); check_format_erasure_values(&formats, set_drive_count)?; if first_disk && DiskError::should_init_erasure_disks(&errs) { // UnformattedDisk, not format file create // new format and save - let fms = init_format_files(disks, set_count, set_drive_count, deployment_id); + let fms = init_format_erasure(disks, set_count, set_drive_count, deployment_id); let _errs = save_format_file_all(disks, &fms).await; // TODO: check quorum // reduceWriteQuorumErrs(&errs)?; - let fm = get_format_file_in_quorum(&fms)?; + let fm = get_format_erasure_in_quorum(&fms)?; return Ok(fm); } @@ -84,12 +84,12 @@ pub async fn do_init_format_file( return Err(Error::new(ErasureError::FirstDiskWait)); } - let fm = get_format_file_in_quorum(&formats)?; + let fm = get_format_erasure_in_quorum(&formats)?; Ok(fm) } -fn init_format_files( +fn init_format_erasure( disks: &[Option], set_count: usize, set_drive_count: usize, @@ -113,7 +113,7 @@ fn init_format_files( fms } -fn get_format_file_in_quorum(formats: &[Option]) -> Result { +fn get_format_erasure_in_quorum(formats: &[Option]) -> Result { let mut countmap = HashMap::new(); for f in formats.iter() { @@ -131,7 +131,7 @@ fn get_format_file_in_quorum(formats: &[Option]) -> Result { let (max_drives, max_count) = countmap.iter().max_by_key(|&(_, c)| c).unwrap_or((&0, &0)); - warn!("get_format_file_in_quorum fi: {:?}", &formats); + warn!("get_format_erasure_in_quorum fi: {:?}", &formats); if *max_drives == 0 || *max_count < formats.len() / 2 { warn!( @@ -198,8 +198,8 @@ pub fn default_partiy_count(drive: usize) -> usize { _ => 4, } } -// read_format_file_all 读取所有foramt.json -async fn read_format_file_all(disks: &[Option], heal: bool) -> (Vec>, Vec>) { +// load_format_erasure_all 读取所有foramt.json +async fn load_format_erasure_all(disks: &[Option], heal: bool) -> (Vec>, Vec>) { let mut futures = Vec::with_capacity(disks.len()); for disk in disks.iter() { From a2b7c4b245606f0819ffaca9e95d0864954c5c3e Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 11 Sep 2024 16:27:04 +0800 Subject: [PATCH 21/23] update fn name --- ecstore/src/store.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index a0303874..2296fa8d 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -183,7 +183,7 @@ impl ECStore { DiskError::check_disk_fatal_errs(&errs)?; - let fm = store_init::do_init_format_file( + let fm = store_init::connect_load_init_formats( first_is_local, &disks, pool_eps.set_count, From 2339bdcbdff8b971c4ae2136d26033af80b240ca Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 11 Sep 2024 16:27:39 +0800 Subject: [PATCH 22/23] update fn name --- ecstore/src/store.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index 2296fa8d..f8ab6cc7 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -192,6 +192,8 @@ impl ECStore { ) .await?; + // TODO: 失败 重试 等试 3次 + if deployment_id.is_none() { deployment_id = Some(fm.id); } From ff053c36e5e77db791e871f7baf0e7250060de6a Mon Sep 17 00:00:00 2001 From: junxiang Mu <1948535941@qq.com> Date: Wed, 11 Sep 2024 11:21:01 +0800 Subject: [PATCH 23/23] support multi node Signed-off-by: junxiang Mu <1948535941@qq.com> --- Cargo.lock | 98 +++++++- Cargo.toml | 1 + .../src/generated/proto_gen/node_service.rs | 99 ++++++++ common/protos/src/node.proto | 14 ++ coordinator/Cargo.toml | 15 -- coordinator/src/lib.rs | 5 - e2e_test/src/reliant/node_interact_test.rs | 22 +- ecstore/Cargo.toml | 3 +- ecstore/src/disk/mod.rs | 17 +- ecstore/src/disk/remote.rs | 227 ++++++++++++++---- ecstore/src/peer.rs | 88 ++++--- ecstore/src/store.rs | 30 ++- ecstore/src/store_init.rs | 2 +- rustfs/src/grpc.rs | 77 +++++- rustfs/src/main.rs | 1 + 15 files changed, 562 insertions(+), 137 deletions(-) delete mode 100644 coordinator/Cargo.toml delete mode 100644 coordinator/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index a7a81c8a..a30a9562 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -94,9 +94,9 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.86" +version = "1.0.88" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b3d1d046238990b9cf5bcde22a3fb3584ee5cf65fb2765f454ed428c7a0063da" +checksum = "4e1496f8fb1fbf272686b8d37f523dab3e4a7443300055e74cdaa449f3114356" [[package]] name = "arrayvec" @@ -205,6 +205,17 @@ dependencies = [ "tower-service", ] +[[package]] +name = "backon" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4fa97bb310c33c811334143cf64c5bb2b7b3c06e453db6b095d7061eff8f113" +dependencies = [ + "fastrand", + "gloo-timers", + "tokio", +] + [[package]] name = "backtrace" version = "0.3.73" @@ -257,6 +268,12 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bumpalo" +version = "3.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79296716171880943b8470b5f8d03aa55eb2e645a4874bdbb28adb49162e012c" + [[package]] name = "byteorder" version = "1.5.0" @@ -420,6 +437,7 @@ name = "ecstore" version = "0.1.0" dependencies = [ "async-trait", + "backon", "base64-simd", "byteorder", "bytes", @@ -657,6 +675,18 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40ecd4077b5ae9fd2e9e169b102c6c330d0605168eb0e8bf79952b256dbefffd" +[[package]] +name = "gloo-timers" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbb143cf96099802033e0d4f4963b19fd2e0b728bcf076cd9cf7f6634f092994" +dependencies = [ + "futures-channel", + "futures-core", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "h2" version = "0.4.5" @@ -882,6 +912,15 @@ version = "1.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49f1f14873335454500d59611f1cf4a4b0f786f9ac11f4312a78e4cf2566695b" +[[package]] +name = "js-sys" +version = "0.3.70" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1868808506b929d7b0cfa8f75951347aa71bb21144b7791bae35d9bccfcfe37a" +dependencies = [ + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.5.0" @@ -2291,6 +2330,61 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a82edfc16a6c469f5f44dc7b571814045d60404b55a0ee849f9bcfa2e63dd9b5" +dependencies = [ + "cfg-if", + "once_cell", + "wasm-bindgen-macro", +] + +[[package]] +name = "wasm-bindgen-backend" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9de396da306523044d3302746f1208fa71d7532227f15e347e2d93e4145dd77b" +dependencies = [ + "bumpalo", + "log", + "once_cell", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "585c4c91a46b072c92e908d99cb1dcdf95c5218eeb6f3bf1efa991ee7a68cccf" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "afc340c74d9005395cf9dd098506f7f44e38f2b4a21c6aaacf9a105ea5e1e836" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-backend", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.93" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c62a0a307cb4a311d3a07867860911ca130c3494e8c2719593806c08bc5d0484" + [[package]] name = "winapi" version = "0.3.9" diff --git a/Cargo.toml b/Cargo.toml index 88dc7fa8..003fce5d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,6 +11,7 @@ version = "0.0.1" [workspace.dependencies] async-trait = "0.1.80" +backon = "1.2.0" bytes = "1.6.0" clap = { version = "4.5.7", features = ["derive"] } ecstore = { path = "./ecstore" } diff --git a/common/protos/src/generated/proto_gen/node_service.rs b/common/protos/src/generated/proto_gen/node_service.rs index 5e677b39..b44e1c1d 100644 --- a/common/protos/src/generated/proto_gen/node_service.rs +++ b/common/protos/src/generated/proto_gen/node_service.rs @@ -419,6 +419,28 @@ pub struct ReadXlResponse { } #[allow(clippy::derive_partial_eq_without_eq)] #[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteVersionsRequest { + #[prost(string, tag = "1")] + pub disk: ::prost::alloc::string::String, + #[prost(string, tag = "2")] + pub volume: ::prost::alloc::string::String, + #[prost(string, repeated, tag = "3")] + pub versions: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, tag = "4")] + pub opts: ::prost::alloc::string::String, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] +pub struct DeleteVersionsResponse { + #[prost(bool, tag = "1")] + pub success: bool, + #[prost(string, repeated, tag = "2")] + pub errors: ::prost::alloc::vec::Vec<::prost::alloc::string::String>, + #[prost(string, optional, tag = "3")] + pub error_info: ::core::option::Option<::prost::alloc::string::String>, +} +#[allow(clippy::derive_partial_eq_without_eq)] +#[derive(Clone, PartialEq, ::prost::Message)] pub struct ReadMultipleRequest { #[prost(string, tag = "1")] pub disk: ::prost::alloc::string::String, @@ -1048,6 +1070,31 @@ pub mod node_service_client { .insert(GrpcMethod::new("node_service.NodeService", "ReadXL")); self.inner.unary(req, path, codec).await } + pub async fn delete_versions( + &mut self, + request: impl tonic::IntoRequest, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + > { + self.inner + .ready() + .await + .map_err(|e| { + tonic::Status::new( + tonic::Code::Unknown, + format!("Service was not ready: {}", e.into()), + ) + })?; + let codec = tonic::codec::ProstCodec::default(); + let path = http::uri::PathAndQuery::from_static( + "/node_service.NodeService/DeleteVersions", + ); + let mut req = request.into_request(); + req.extensions_mut() + .insert(GrpcMethod::new("node_service.NodeService", "DeleteVersions")); + self.inner.unary(req, path, codec).await + } pub async fn read_multiple( &mut self, request: impl tonic::IntoRequest, @@ -1232,6 +1279,13 @@ pub mod node_service_server { &self, request: tonic::Request, ) -> std::result::Result, tonic::Status>; + async fn delete_versions( + &self, + request: tonic::Request, + ) -> std::result::Result< + tonic::Response, + tonic::Status, + >; async fn read_multiple( &self, request: tonic::Request, @@ -2264,6 +2318,51 @@ pub mod node_service_server { }; Box::pin(fut) } + "/node_service.NodeService/DeleteVersions" => { + #[allow(non_camel_case_types)] + struct DeleteVersionsSvc(pub Arc); + impl< + T: NodeService, + > tonic::server::UnaryService + for DeleteVersionsSvc { + type Response = super::DeleteVersionsResponse; + type Future = BoxFuture< + tonic::Response, + tonic::Status, + >; + fn call( + &mut self, + request: tonic::Request, + ) -> Self::Future { + let inner = Arc::clone(&self.0); + let fut = async move { + ::delete_versions(&inner, request).await + }; + Box::pin(fut) + } + } + let accept_compression_encodings = self.accept_compression_encodings; + let send_compression_encodings = self.send_compression_encodings; + let max_decoding_message_size = self.max_decoding_message_size; + let max_encoding_message_size = self.max_encoding_message_size; + let inner = self.inner.clone(); + let fut = async move { + let method = DeleteVersionsSvc(inner); + let codec = tonic::codec::ProstCodec::default(); + let mut grpc = tonic::server::Grpc::new(codec) + .apply_compression_config( + accept_compression_encodings, + send_compression_encodings, + ) + .apply_max_message_size_config( + max_decoding_message_size, + max_encoding_message_size, + ); + let res = grpc.unary(method, req).await; + Ok(res) + }; + Box::pin(fut) + } "/node_service.NodeService/ReadMultiple" => { #[allow(non_camel_case_types)] struct ReadMultipleSvc(pub Arc); diff --git a/common/protos/src/node.proto b/common/protos/src/node.proto index 1d7fe716..a7e64b0a 100644 --- a/common/protos/src/node.proto +++ b/common/protos/src/node.proto @@ -258,6 +258,19 @@ message ReadXLResponse { optional string error_info = 3; } +message DeleteVersionsRequest { + string disk = 1; + string volume = 2; + repeated string versions = 3; + string opts = 4; +} + +message DeleteVersionsResponse { + bool success = 1; + repeated string errors = 2; + optional string error_info = 3; +} + message ReadMultipleRequest { string disk = 1; string read_multiple_req = 2; @@ -308,6 +321,7 @@ service NodeService { rpc WriteMetadata(WriteMetadataRequest) returns (WriteMetadataResponse) {}; rpc ReadVersion(ReadVersionRequest) returns (ReadVersionResponse) {}; rpc ReadXL(ReadXLRequest) returns (ReadXLResponse) {}; + rpc DeleteVersions(DeleteVersionsRequest) returns (DeleteVersionsResponse) {}; rpc ReadMultiple(ReadMultipleRequest) returns (ReadMultipleResponse) {}; rpc DeleteVolume(DeleteVolumeRequest) returns (DeleteVolumeResponse) {}; } diff --git a/coordinator/Cargo.toml b/coordinator/Cargo.toml deleted file mode 100644 index a1e30a55..00000000 --- a/coordinator/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "e2e_test" -version.workspace = true -edition.workspace = true -license.workspace = true -repository.workspace = true -rust-version.workspace = true - -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - -[dependencies] -flatbuffers.workspace = true -protos.workspace = true -tonic = { version = "0.12.1", features = ["gzip"] } -tokio = { workspace = true } \ No newline at end of file diff --git a/coordinator/src/lib.rs b/coordinator/src/lib.rs deleted file mode 100644 index 10439ca7..00000000 --- a/coordinator/src/lib.rs +++ /dev/null @@ -1,5 +0,0 @@ -#[async_trait::async_trait] -pub trait Coordinator: Send + Sync { - async fn ping(); - async fn create_bucket() -> Result<> -} \ No newline at end of file diff --git a/e2e_test/src/reliant/node_interact_test.rs b/e2e_test/src/reliant/node_interact_test.rs index ca6930e8..d0566c3f 100644 --- a/e2e_test/src/reliant/node_interact_test.rs +++ b/e2e_test/src/reliant/node_interact_test.rs @@ -4,13 +4,14 @@ use ecstore::disk::VolumeInfo; use protos::{ models::{PingBody, PingBodyBuilder}, proto_gen::node_service::{ - node_service_client::NodeServiceClient, ListVolumesRequest, MakeVolumeRequest, PingRequest, PingResponse, + node_service_client::NodeServiceClient, ListVolumesRequest, MakeVolumeRequest, PingRequest, PingResponse, ReadAllRequest, }, }; use std::error::Error; use tonic::Request; async fn get_client() -> Result, Box> { + // Ok(NodeServiceClient::connect("http://220.181.1.138:9000").await?) Ok(NodeServiceClient::connect("http://localhost:9000").await?) } @@ -30,7 +31,7 @@ async fn ping() -> Result<(), Box> { assert!(decoded_payload.is_ok()); // 创建客户端 - let mut client = NodeServiceClient::connect("http://localhost:9000").await?; + let mut client = get_client().await?; // 构造 PingRequest let request = Request::new(PingRequest { @@ -86,3 +87,20 @@ async fn list_volumes() -> Result<(), Box> { println!("{:?}", volume_infos); Ok(()) } + +#[tokio::test] +async fn read_all() -> Result<(), Box> { + let mut client = get_client().await?; + let request = Request::new(ReadAllRequest { + disk: "data".to_string(), + volume: "ff".to_string(), + path: "format.json".to_string(), + }); + + let response = client.read_all(request).await?.into_inner(); + let volume_infos = response.data; + + println!("{}", response.success); + println!("{:?}", volume_infos); + Ok(()) +} diff --git a/ecstore/Cargo.toml b/ecstore/Cargo.toml index de373425..ada0fc7a 100644 --- a/ecstore/Cargo.toml +++ b/ecstore/Cargo.toml @@ -9,7 +9,7 @@ rust-version.workspace = true # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -tokio = { workspace = true, features = ["io-util"] } +backon.workspace = true bytes.workspace = true thiserror.workspace = true futures.workspace = true @@ -38,6 +38,7 @@ base64-simd = "0.8.0" sha2 = "0.10.8" hex-simd = "0.8.0" path-clean = "1.0.1" +tokio = { workspace = true, features = ["io-util"] } tokio-stream = "0.1.15" tonic.workspace = true tower.workspace = true diff --git a/ecstore/src/disk/mod.rs b/ecstore/src/disk/mod.rs index 19b8a104..d2f63c4f 100644 --- a/ecstore/src/disk/mod.rs +++ b/ecstore/src/disk/mod.rs @@ -28,7 +28,6 @@ use tokio::{ io::{AsyncReadExt, AsyncSeekExt, AsyncWriteExt}, }; use tonic::{transport::Channel, Request}; -use tower::timeout::Timeout; use uuid::Uuid; pub type DiskStore = Arc>; @@ -96,7 +95,7 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn read_multiple(&self, req: ReadMultipleReq) -> Result>; } -#[derive(Debug, Default, Clone)] +#[derive(Debug, Default, Clone, Serialize, Deserialize)] pub struct FileInfoVersions { // Name of the volume. pub volume: String, @@ -350,17 +349,11 @@ pub struct RemoteFileWriter { pub volume: String, pub path: String, pub is_append: bool, - client: NodeServiceClient>, + client: NodeServiceClient, } impl RemoteFileWriter { - pub fn new( - root: PathBuf, - volume: String, - path: String, - is_append: bool, - client: NodeServiceClient>, - ) -> Self { + pub fn new(root: PathBuf, volume: String, path: String, is_append: bool, client: NodeServiceClient) -> Self { Self { root, volume, @@ -433,11 +426,11 @@ pub struct RemoteFileReader { pub root: PathBuf, pub volume: String, pub path: String, - client: NodeServiceClient>, + client: NodeServiceClient, } impl RemoteFileReader { - pub fn new(root: PathBuf, volume: String, path: String, client: NodeServiceClient>) -> Self { + pub fn new(root: PathBuf, volume: String, path: String, client: NodeServiceClient) -> Self { Self { root, volume, diff --git a/ecstore/src/disk/remote.rs b/ecstore/src/disk/remote.rs index 2024d04b..b0d5a8a0 100644 --- a/ecstore/src/disk/remote.rs +++ b/ecstore/src/disk/remote.rs @@ -1,17 +1,18 @@ -use std::{path::PathBuf, sync::RwLock, time::Duration}; +use std::{path::PathBuf, sync::Arc, time::Duration}; use bytes::Bytes; use futures::lock::Mutex; use protos::{ node_service_time_out_client, proto_gen::node_service::{ - node_service_client::NodeServiceClient, DeleteRequest, DeleteVolumeRequest, ListDirRequest, ListVolumesRequest, - MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, ReadXlRequest, - RenameDataRequest, RenameFileRequst, StatVolumeRequest, WalkDirRequest, WriteAllRequest, WriteMetadataRequest, + node_service_client::NodeServiceClient, DeleteRequest, DeleteVersionsRequest, DeleteVolumeRequest, ListDirRequest, + ListVolumesRequest, MakeVolumeRequest, MakeVolumesRequest, ReadAllRequest, ReadMultipleRequest, ReadVersionRequest, + ReadXlRequest, RenameDataRequest, RenameFileRequst, StatVolumeRequest, WalkDirRequest, WriteAllRequest, + WriteMetadataRequest, }, DEFAULT_GRPC_SERVER_MESSAGE_LEN, }; -use tokio::fs; +use tokio::{fs, sync::RwLock}; use tonic::{ transport::{Channel, Endpoint as tonic_Endpoint}, Request, @@ -21,6 +22,7 @@ use tracing::info; use uuid::Uuid; use crate::{ + disk::error::DiskError, error::{Error, Result}, store_api::{FileInfo, RawFileInfo}, }; @@ -34,7 +36,7 @@ use super::{ #[derive(Debug)] pub struct RemoteDisk { id: Mutex>, - channel: RwLock>, + channel: Arc>>, url: url::Url, pub root: PathBuf, } @@ -44,39 +46,49 @@ impl RemoteDisk { let root = fs::canonicalize(ep.url.path()).await?; Ok(Self { - channel: RwLock::new(None), + channel: Arc::new(RwLock::new(None)), url: ep.url.clone(), root, id: Mutex::new(None), }) } - fn get_client(&self) -> NodeServiceClient> { + #[allow(dead_code)] + async fn get_client(&self) -> Result>> { + let channel_clone = self.channel.clone(); let channel = { - let read_lock = self.channel.read().unwrap(); + let read_lock = channel_clone.read().await; if let Some(ref channel) = *read_lock { channel.clone() } else { let addr = format!("{}://{}:{}", self.url.scheme(), self.url.host_str().unwrap(), self.url.port().unwrap()); - info!("disk url: {:?}", addr); - let connector = tonic_Endpoint::from_shared(addr).unwrap(); + info!("disk url: {}", addr); + let connector = tonic_Endpoint::from_shared(addr.clone())?; - let new_channel = tokio::runtime::Runtime::new().unwrap().block_on(connector.connect()).unwrap(); + let new_channel = connector.connect().await.map_err(|_err| DiskError::DiskNotFound)?; - *self.channel.write().unwrap() = Some(new_channel.clone()); + info!("get channel success"); + + *self.channel.write().await = Some(new_channel.clone()); new_channel } }; - node_service_time_out_client( + Ok(node_service_time_out_client( channel, Duration::new(30, 0), // TODO: use config setting DEFAULT_GRPC_SERVER_MESSAGE_LEN, // grpc_enable_gzip, false, // TODO: use config setting - ) + )) + } + + async fn get_client_v2(&self) -> Result> { + // Ok(NodeServiceClient::connect("http://220.181.1.138:9000").await?) + let addr = format!("{}://{}:{}", self.url.scheme(), self.url.host_str().unwrap(), self.url.port().unwrap()); + Ok(NodeServiceClient::connect(addr).await?) } } @@ -104,7 +116,8 @@ impl DiskAPI for RemoteDisk { } async fn read_all(&self, volume: &str, path: &str) -> Result { - let mut client = self.get_client(); + info!("read_all"); + let mut client = self.get_client_v2().await?; let request = Request::new(ReadAllRequest { disk: self.root.to_string_lossy().to_string(), volume: volume.to_string(), @@ -113,11 +126,18 @@ impl DiskAPI for RemoteDisk { let response = client.read_all(request).await?.into_inner(); + info!("read_all success"); + + if !response.success { + return Err(DiskError::FileNotFound.into()); + } + Ok(Bytes::from(response.data)) } async fn write_all(&self, volume: &str, path: &str, data: Vec) -> Result<()> { - let mut client = self.get_client(); + info!("write_all"); + let mut client = self.get_client_v2().await?; let request = Request::new(WriteAllRequest { disk: self.root.to_string_lossy().to_string(), volume: volume.to_string(), @@ -125,14 +145,19 @@ impl DiskAPI for RemoteDisk { data, }); - let _response = client.write_all(request).await?.into_inner(); + let response = client.write_all(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } Ok(()) } async fn delete(&self, volume: &str, path: &str, opt: DeleteOptions) -> Result<()> { + info!("delete"); let options = serde_json::to_string(&opt)?; - let mut client = self.get_client(); + let mut client = self.get_client_v2().await?; let request = Request::new(DeleteRequest { disk: self.root.to_string_lossy().to_string(), volume: volume.to_string(), @@ -140,13 +165,18 @@ impl DiskAPI for RemoteDisk { options, }); - let _response = client.delete(request).await?.into_inner(); + let response = client.delete(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } Ok(()) } async fn rename_file(&self, src_volume: &str, src_path: &str, dst_volume: &str, dst_path: &str) -> Result<()> { - let mut client = self.get_client(); + info!("rename_file"); + let mut client = self.get_client_v2().await?; let request = Request::new(RenameFileRequst { disk: self.root.to_string_lossy().to_string(), src_volume: src_volume.to_string(), @@ -155,42 +185,50 @@ impl DiskAPI for RemoteDisk { dst_path: dst_path.to_string(), }); - let _response = client.rename_file(request).await?.into_inner(); + let response = client.rename_file(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } Ok(()) } async fn create_file(&self, _origvolume: &str, volume: &str, path: &str, _file_size: usize) -> Result { + info!("create_file"); Ok(FileWriter::Remote(RemoteFileWriter::new( self.root.clone(), volume.to_string(), path.to_string(), false, - self.get_client(), + self.get_client_v2().await?, ))) } async fn append_file(&self, volume: &str, path: &str) -> Result { + info!("append_file"); Ok(FileWriter::Remote(RemoteFileWriter::new( self.root.clone(), volume.to_string(), path.to_string(), true, - self.get_client(), + self.get_client_v2().await?, ))) } async fn read_file(&self, volume: &str, path: &str) -> Result { + info!("read_file"); Ok(FileReader::Remote(RemoteFileReader::new( self.root.clone(), volume.to_string(), path.to_string(), - self.get_client(), + self.get_client_v2().await?, ))) } async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result> { - let mut client = self.get_client(); + info!("list_dir"); + let mut client = self.get_client_v2().await?; let request = Request::new(ListDirRequest { disk: self.root.to_string_lossy().to_string(), volume: volume.to_string(), @@ -198,18 +236,28 @@ impl DiskAPI for RemoteDisk { let response = client.list_dir(request).await?.into_inner(); + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + Ok(response.volumes) } async fn walk_dir(&self, opts: WalkDirOptions) -> Result> { + info!("walk_dir"); let walk_dir_options = serde_json::to_string(&opts)?; - let mut client = self.get_client(); + let mut client = self.get_client_v2().await?; let request = Request::new(WalkDirRequest { disk: self.root.to_string_lossy().to_string(), walk_dir_options, }); let response = client.walk_dir(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + let entries = response .meta_cache_entry .into_iter() @@ -227,8 +275,9 @@ impl DiskAPI for RemoteDisk { dst_volume: &str, dst_path: &str, ) -> Result { + info!("rename_data"); let file_info = serde_json::to_string(&fi)?; - let mut client = self.get_client(); + let mut client = self.get_client_v2().await?; let request = Request::new(RenameDataRequest { disk: self.root.to_string_lossy().to_string(), src_volume: src_volume.to_string(), @@ -239,42 +288,63 @@ impl DiskAPI for RemoteDisk { }); let response = client.rename_data(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + let rename_data_resp = serde_json::from_str::(&response.rename_data_resp)?; Ok(rename_data_resp) } async fn make_volumes(&self, volumes: Vec<&str>) -> Result<()> { - let mut client = self.get_client(); + info!("make_volumes"); + let mut client = self.get_client_v2().await?; let request = Request::new(MakeVolumesRequest { disk: self.root.to_string_lossy().to_string(), volumes: volumes.iter().map(|s| (*s).to_string()).collect(), }); - let _response = client.make_volumes(request).await?.into_inner(); + let response = client.make_volumes(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } Ok(()) } async fn make_volume(&self, volume: &str) -> Result<()> { - let mut client = self.get_client(); + info!("make_volume"); + let mut client = self.get_client_v2().await?; let request = Request::new(MakeVolumeRequest { disk: self.root.to_string_lossy().to_string(), volume: volume.to_string(), }); - let _response = client.make_volume(request).await?.into_inner(); + let response = client.make_volume(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } Ok(()) } async fn list_volumes(&self) -> Result> { - let mut client = self.get_client(); + info!("list_volumes"); + let mut client = self.get_client_v2().await?; let request = Request::new(ListVolumesRequest { disk: self.root.to_string_lossy().to_string(), }); let response = client.list_volumes(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + let infos = response .volume_infos .into_iter() @@ -285,21 +355,28 @@ impl DiskAPI for RemoteDisk { } async fn stat_volume(&self, volume: &str) -> Result { - let mut client = self.get_client(); + info!("stat_volume"); + let mut client = self.get_client_v2().await?; let request = Request::new(StatVolumeRequest { disk: self.root.to_string_lossy().to_string(), volume: volume.to_string(), }); let response = client.stat_volume(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + let volume_info = serde_json::from_str::(&response.volume_info)?; Ok(volume_info) } async fn write_metadata(&self, _org_volume: &str, volume: &str, path: &str, fi: FileInfo) -> Result<()> { + info!("write_metadata"); let file_info = serde_json::to_string(&fi)?; - let mut client = self.get_client(); + let mut client = self.get_client_v2().await?; let request = Request::new(WriteMetadataRequest { disk: self.root.to_string_lossy().to_string(), volume: volume.to_string(), @@ -307,7 +384,11 @@ impl DiskAPI for RemoteDisk { file_info, }); - let _response = client.write_metadata(request).await?.into_inner(); + let response = client.write_metadata(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } Ok(()) } @@ -320,8 +401,9 @@ impl DiskAPI for RemoteDisk { version_id: &str, opts: &ReadOptions, ) -> Result { + info!("read_version"); let opts = serde_json::to_string(opts)?; - let mut client = self.get_client(); + let mut client = self.get_client_v2().await?; let request = Request::new(ReadVersionRequest { disk: self.root.to_string_lossy().to_string(), volume: volume.to_string(), @@ -331,13 +413,19 @@ impl DiskAPI for RemoteDisk { }); let response = client.read_version(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + let file_info = serde_json::from_str::(&response.file_info)?; Ok(file_info) } async fn read_xl(&self, volume: &str, path: &str, read_data: bool) -> Result { - let mut client = self.get_client(); + info!("read_xl"); + let mut client = self.get_client_v2().await?; let request = Request::new(ReadXlRequest { disk: self.root.to_string_lossy().to_string(), volume: volume.to_string(), @@ -346,6 +434,11 @@ impl DiskAPI for RemoteDisk { }); let response = client.read_xl(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + let raw_file_info = serde_json::from_str::(&response.raw_file_info)?; Ok(raw_file_info) @@ -353,22 +446,61 @@ impl DiskAPI for RemoteDisk { async fn delete_versions( &self, - _volume: &str, - _versions: Vec, - _opts: DeleteOptions, + volume: &str, + versions: Vec, + opts: DeleteOptions, ) -> Result>> { - unimplemented!() + info!("delete_versions"); + let opts = serde_json::to_string(&opts)?; + let mut versions_str = Vec::with_capacity(versions.len()); + for file_info_versions in versions.iter() { + versions_str.push(serde_json::to_string(file_info_versions)?); + } + let mut client = self.get_client_v2().await?; + let request = Request::new(DeleteVersionsRequest { + disk: self.root.to_string_lossy().to_string(), + volume: volume.to_string(), + versions: versions_str, + opts, + }); + + let response = client.delete_versions(request).await?.into_inner(); + if !response.success { + return Err(Error::from_string(format!( + "delete versions remote err: {}", + response.error_info.unwrap_or("None".to_string()) + ))); + } + let errors = response + .errors + .iter() + .map(|error| { + if error.is_empty() { + None + } else { + Some(Error::from_string(error)) + } + }) + .collect(); + + Ok(errors) } async fn read_multiple(&self, req: ReadMultipleReq) -> Result> { + info!("read_multiple"); let read_multiple_req = serde_json::to_string(&req)?; - let mut client = self.get_client(); + let mut client = self.get_client_v2().await?; let request = Request::new(ReadMultipleRequest { disk: self.root.to_string_lossy().to_string(), read_multiple_req, }); let response = client.read_multiple(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } + let read_multiple_resps = response .read_multiple_resps .into_iter() @@ -379,13 +511,18 @@ impl DiskAPI for RemoteDisk { } async fn delete_volume(&self, volume: &str) -> Result<()> { - let mut client = self.get_client(); + info!("delete_volume"); + let mut client = self.get_client_v2().await?; let request = Request::new(DeleteVolumeRequest { disk: self.root.to_string_lossy().to_string(), volume: volume.to_string(), }); - let _response = client.delete_volume(request).await?.into_inner(); + let response = client.delete_volume(request).await?.into_inner(); + + if !response.success { + return Err(Error::from_string(response.error_info.unwrap_or("".to_string()))); + } Ok(()) } diff --git a/ecstore/src/peer.rs b/ecstore/src/peer.rs index e2c8f163..4588d335 100644 --- a/ecstore/src/peer.rs +++ b/ecstore/src/peer.rs @@ -1,12 +1,15 @@ use async_trait::async_trait; use futures::future::join_all; +use protos::proto_gen::node_service::node_service_client::NodeServiceClient; use protos::proto_gen::node_service::{DeleteBucketRequest, GetBucketInfoRequest, ListBucketRequest, MakeBucketRequest}; use protos::{node_service_time_out_client, DEFAULT_GRPC_SERVER_MESSAGE_LEN}; use regex::Regex; use std::{collections::HashMap, fmt::Debug, sync::Arc, time::Duration}; +use tokio::sync::RwLock; use tonic::transport::{Channel, Endpoint}; use tonic::Request; -use tracing::warn; +use tower::timeout::Timeout; +use tracing::{info, warn}; use crate::store::all_local_disk; use crate::{ @@ -386,22 +389,58 @@ impl PeerS3Client for LocalPeerS3Client { #[derive(Debug)] pub struct RemotePeerS3Client { - pub _node: Option, + pub node: Option, pub pools: Option>, - pub channel: Channel, + connector: Endpoint, + channel: Arc>>, } impl RemotePeerS3Client { fn new(node: Option, pools: Option>) -> Self { let connector = Endpoint::from_shared(format!("{}", node.as_ref().map(|v| { v.url.to_string() }).unwrap_or_default())).unwrap(); - let channel = tokio::runtime::Runtime::new().unwrap().block_on(connector.connect()).unwrap(); Self { - _node: node, + node, pools, - channel, + connector, + channel: Arc::new(RwLock::new(None)), } } + + #[allow(dead_code)] + async fn get_client(&self) -> Result>> { + let channel_clone = self.channel.clone(); + let channel = { + let read_lock = channel_clone.read().await; + + if let Some(ref channel) = *read_lock { + channel.clone() + } else { + let new_channel = self.connector.connect().await?; + + info!("get channel success"); + + *self.channel.write().await = Some(new_channel.clone()); + + new_channel + } + }; + + Ok(node_service_time_out_client( + channel, + Duration::new(30, 0), // TODO: use config setting + DEFAULT_GRPC_SERVER_MESSAGE_LEN, + // grpc_enable_gzip, + false, // TODO: use config setting + )) + } + + async fn get_client_v2(&self) -> Result> { + // Ok(NodeServiceClient::connect("http://220.181.1.138:9000").await?) + // let addr = format!("{}://{}:{}", self.url.scheme(), self.url.host_str().unwrap(), self.url.port().unwrap()); + let addr = format!("{}", self.node.as_ref().map(|v| { v.url.to_string() }).unwrap_or_default()); + Ok(NodeServiceClient::connect(addr).await?) + } } #[async_trait] @@ -411,13 +450,7 @@ impl PeerS3Client for RemotePeerS3Client { } async fn list_bucket(&self, opts: &BucketOptions) -> Result> { let options = serde_json::to_string(opts)?; - let mut client = node_service_time_out_client( - self.channel.clone(), - Duration::new(30, 0), // TODO: use config setting - DEFAULT_GRPC_SERVER_MESSAGE_LEN, - // grpc_enable_gzip, - false, // TODO: use config setting - ); + let mut client = self.get_client_v2().await?; let request = Request::new(ListBucketRequest { options }); let response = client.list_bucket(request).await?.into_inner(); let bucket_infos = response @@ -430,32 +463,23 @@ impl PeerS3Client for RemotePeerS3Client { } async fn make_bucket(&self, bucket: &str, opts: &MakeBucketOptions) -> Result<()> { let options = serde_json::to_string(opts)?; - let mut client = node_service_time_out_client( - self.channel.clone(), - Duration::new(30, 0), // TODO: use config setting - DEFAULT_GRPC_SERVER_MESSAGE_LEN, - // grpc_enable_gzip, - false, // TODO: use config setting - ); + let mut client = self.get_client_v2().await?; let request = Request::new(MakeBucketRequest { name: bucket.to_string(), options, }); - let _response = client.make_bucket(request).await?.into_inner(); + let response = client.make_bucket(request).await?.into_inner(); // TODO: deal with error + if !response.success { + warn!("make bucket error: {:?}", response.error_info); + } Ok(()) } async fn get_bucket_info(&self, bucket: &str, opts: &BucketOptions) -> Result { let options = serde_json::to_string(opts)?; - let mut client = node_service_time_out_client( - self.channel.clone(), - Duration::new(30, 0), // TODO: use config setting - DEFAULT_GRPC_SERVER_MESSAGE_LEN, - // grpc_enable_gzip, - false, // TODO: use config setting - ); + let mut client = self.get_client_v2().await?; let request = Request::new(GetBucketInfoRequest { bucket: bucket.to_string(), options, @@ -467,13 +491,7 @@ impl PeerS3Client for RemotePeerS3Client { } async fn delete_bucket(&self, bucket: &str) -> Result<()> { - let mut client = node_service_time_out_client( - self.channel.clone(), - Duration::new(30, 0), // TODO: use config setting - DEFAULT_GRPC_SERVER_MESSAGE_LEN, - // grpc_enable_gzip, - false, // TODO: use config setting - ); + let mut client = self.get_client_v2().await?; let request = Request::new(DeleteBucketRequest { bucket: bucket.to_string(), }); diff --git a/ecstore/src/store.rs b/ecstore/src/store.rs index f8ab6cc7..41219bf7 100644 --- a/ecstore/src/store.rs +++ b/ecstore/src/store.rs @@ -14,16 +14,18 @@ use crate::{ }, store_init, utils, }; +use backon::{ExponentialBuilder, Retryable}; use futures::future::join_all; use http::HeaderMap; use s3s::{dto::StreamingBlob, Body}; use std::{ collections::{HashMap, HashSet}, sync::Arc, + time::Duration, }; use time::OffsetDateTime; use tokio::{fs, sync::RwLock}; -use tracing::{debug, warn}; +use tracing::{debug, info, warn}; use uuid::Uuid; use lazy_static::lazy_static; @@ -168,6 +170,8 @@ impl ECStore { let mut local_disks = Vec::new(); + info!("endpoint_pools: {:?}", endpoint_pools); + for (i, pool_eps) in endpoint_pools.as_ref().iter().enumerate() { // TODO: read from config parseStorageClass let partiy_count = store_init::default_partiy_count(pool_eps.drives_per_set); @@ -183,17 +187,23 @@ impl ECStore { DiskError::check_disk_fatal_errs(&errs)?; - let fm = store_init::connect_load_init_formats( - first_is_local, - &disks, - pool_eps.set_count, - pool_eps.drives_per_set, - deployment_id, - ) + let fm = (|| async { + store_init::connect_load_init_formats( + first_is_local, + &disks, + pool_eps.set_count, + pool_eps.drives_per_set, + deployment_id, + ) + .await + }) + .retry(ExponentialBuilder::default().with_max_times(usize::MAX)) + .sleep(tokio::time::sleep) + .notify(|err, dur: Duration| { + info!("retrying get formats {:?} after {:?}", err, dur); + }) .await?; - // TODO: 失败 重试 等试 3次 - if deployment_id.is_none() { deployment_id = Some(fm.id); } diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index 245f0b27..6e6eb2fc 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -133,7 +133,7 @@ fn get_format_erasure_in_quorum(formats: &[Option]) -> Result) -> Result, Status> { + debug!("read all"); + let request = request.into_inner(); if let Some(disk) = self.find_disk(&request.disk).await { match disk.read_all(&request.volume, &request.path).await { @@ -693,6 +695,63 @@ impl Node for NodeService { } } + async fn delete_versions(&self, request: Request) -> Result, Status> { + let request = request.into_inner(); + if let Some(disk) = self.find_disk(&request.disk).await { + let mut versions = Vec::with_capacity(request.versions.len()); + for version in request.versions.iter() { + match serde_json::from_str::(&version) { + Ok(version) => versions.push(version), + Err(_) => { + return Ok(tonic::Response::new(DeleteVersionsResponse { + success: false, + errors: Vec::new(), + error_info: Some("can not decode FileInfoVersions".to_string()), + })); + } + }; + } + let opts = match serde_json::from_str::(&request.opts) { + Ok(opts) => opts, + Err(_) => { + return Ok(tonic::Response::new(DeleteVersionsResponse { + success: false, + errors: Vec::new(), + error_info: Some("can not decode DeleteOptions".to_string()), + })); + } + }; + match disk.delete_versions(&request.volume, versions, opts).await { + Ok(errors) => { + let errors = errors + .into_iter() + .map(|error| match error { + Some(e) => e.to_string(), + None => "".to_string(), + }) + .collect(); + + Ok(tonic::Response::new(DeleteVersionsResponse { + success: true, + errors, + error_info: None, + })) + } + Err(err) => Ok(tonic::Response::new(DeleteVersionsResponse { + success: false, + errors: Vec::new(), + error_info: Some(err.to_string()), + })), + } + } else { + Ok(tonic::Response::new(DeleteVersionsResponse { + success: false, + errors: Vec::new(), + error_info: Some("can not find disk".to_string()), + })) + } + } + async fn read_multiple(&self, request: Request) -> Result, Status> { let request = request.into_inner(); if let Some(disk) = self.find_disk(&request.disk).await { diff --git a/rustfs/src/main.rs b/rustfs/src/main.rs index a3e962f5..aa51bc44 100644 --- a/rustfs/src/main.rs +++ b/rustfs/src/main.rs @@ -167,6 +167,7 @@ async fn run(opt: config::Opt) -> Result<()> { warn!(" init store"); // init store ECStore::new(opt.address.clone(), endpoint_pools.clone()).await?; + warn!(" init store success!"); tokio::select! { _ = tokio::signal::ctrl_c() => {