From b3c80ae362fc21454f0edeef46af5384bb35e9f0 Mon Sep 17 00:00:00 2001 From: weisd Date: Thu, 4 Dec 2025 16:12:10 +0800 Subject: [PATCH] fix: listdir rpc (#979) Co-authored-by: houseme Co-authored-by: loverustfs --- Cargo.lock | 22 +++--- Cargo.toml | 4 +- crates/ecstore/src/disk/mod.rs | 6 +- crates/ecstore/src/rpc/remote_disk.rs | 8 ++- .../generated/flatbuffers_generated/mod.rs | 14 ++++ .../src/generated/proto_gen/node_service.rs | 4 ++ crates/protos/src/main.rs | 34 +++++++++- crates/protos/src/node.proto | 2 + rustfs/src/storage/tonic_service.rs | 4 +- scripts/install-flatc.sh | 67 +++++++++++++++++++ scripts/install-protoc.sh | 67 +++++++++++++++++++ 11 files changed, 210 insertions(+), 22 deletions(-) create mode 100755 scripts/install-flatc.sh create mode 100755 scripts/install-protoc.sh diff --git a/Cargo.lock b/Cargo.lock index fc475e3e..b9a69113 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -691,9 +691,9 @@ dependencies = [ [[package]] name = "aws-sdk-s3" -version = "1.115.0" +version = "1.116.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fdaa0053cbcbc384443dd24569bd5d1664f86427b9dc04677bd0ab853954baec" +checksum = "cd4c10050aa905b50dc2a1165a9848d598a80c3a724d6f93b5881aa62235e4a5" dependencies = [ "aws-credential-types", "aws-runtime", @@ -4266,9 +4266,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.18" +version = "0.1.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56" +checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f" dependencies = [ "base64 0.22.1", "bytes", @@ -4816,9 +4816,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" +checksum = "8b484ba8d4f775eeca644c452a56650e544bf7e617f1d170fe7298122ead5222" dependencies = [ "zlib-rs", ] @@ -4875,11 +4875,11 @@ dependencies = [ [[package]] name = "log" -version = "0.4.28" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" dependencies = [ - "serde", + "serde_core", "value-bag", ] @@ -10394,9 +10394,9 @@ dependencies = [ [[package]] name = "zlib-rs" -version = "0.5.2" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" +checksum = "36134c44663532e6519d7a6dfdbbe06f6f8192bde8ae9ed076e9b213f0e31df7" [[package]] name = "zopfli" diff --git a/Cargo.toml b/Cargo.toml index 01a089eb..c369f677 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -105,7 +105,7 @@ futures-core = "0.3.31" futures-util = "0.3.31" hyper = { version = "1.8.1", features = ["http2", "http1", "server"] } hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "ring", "webpki-roots"] } -hyper-util = { version = "0.1.18", features = ["tokio", "server-auto", "server-graceful"] } +hyper-util = { version = "0.1.19", features = ["tokio", "server-auto", "server-graceful"] } http = "1.4.0" http-body = "1.0.1" reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2", "system-proxy", "stream", "json", "blocking"] } @@ -167,7 +167,7 @@ atoi = "2.0.0" atomic_enum = "0.3.0" aws-config = { version = "1.8.11" } aws-credential-types = { version = "1.2.10" } -aws-sdk-s3 = { version = "1.115.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] } +aws-sdk-s3 = { version = "1.116.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] } aws-smithy-types = { version = "1.3.4" } base64 = "0.22.1" base64-simd = "0.8.0" diff --git a/crates/ecstore/src/disk/mod.rs b/crates/ecstore/src/disk/mod.rs index 7d301843..5f419380 100644 --- a/crates/ecstore/src/disk/mod.rs +++ b/crates/ecstore/src/disk/mod.rs @@ -271,10 +271,10 @@ impl DiskAPI for Disk { } #[tracing::instrument(skip(self))] - async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result> { + async fn list_dir(&self, _origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result> { match self { - Disk::Local(local_disk) => local_disk.list_dir(_origvolume, volume, _dir_path, _count).await, - Disk::Remote(remote_disk) => remote_disk.list_dir(_origvolume, volume, _dir_path, _count).await, + Disk::Local(local_disk) => local_disk.list_dir(_origvolume, volume, dir_path, count).await, + Disk::Remote(remote_disk) => remote_disk.list_dir(_origvolume, volume, dir_path, count).await, } } diff --git a/crates/ecstore/src/rpc/remote_disk.rs b/crates/ecstore/src/rpc/remote_disk.rs index 99086a88..5e024f0b 100644 --- a/crates/ecstore/src/rpc/remote_disk.rs +++ b/crates/ecstore/src/rpc/remote_disk.rs @@ -42,7 +42,7 @@ use rustfs_protos::proto_gen::node_service::RenamePartRequest; use rustfs_rio::{HttpReader, HttpWriter}; use tokio::{io::AsyncWrite, net::TcpStream, time::timeout}; use tonic::Request; -use tracing::info; +use tracing::{debug, info}; use uuid::Uuid; #[derive(Debug)] @@ -596,14 +596,16 @@ impl DiskAPI for RemoteDisk { } #[tracing::instrument(skip(self))] - async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result> { - info!("list_dir {}/{}", volume, _dir_path); + async fn list_dir(&self, _origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result> { + debug!("list_dir {}/{}", volume, dir_path); let mut client = node_service_time_out_client(&self.addr) .await .map_err(|err| Error::other(format!("can not get client, err: {err}")))?; let request = Request::new(ListDirRequest { disk: self.endpoint.to_string(), volume: volume.to_string(), + dir_path: dir_path.to_string(), + count, }); let response = client.list_dir(request).await?.into_inner(); diff --git a/crates/protos/src/generated/flatbuffers_generated/mod.rs b/crates/protos/src/generated/flatbuffers_generated/mod.rs index c446ac88..ee15f5c7 100644 --- a/crates/protos/src/generated/flatbuffers_generated/mod.rs +++ b/crates/protos/src/generated/flatbuffers_generated/mod.rs @@ -1 +1,15 @@ +// Copyright 2024 RustFS Team +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + pub mod models; diff --git a/crates/protos/src/generated/proto_gen/node_service.rs b/crates/protos/src/generated/proto_gen/node_service.rs index bdde52b8..c7a0ac7f 100644 --- a/crates/protos/src/generated/proto_gen/node_service.rs +++ b/crates/protos/src/generated/proto_gen/node_service.rs @@ -303,6 +303,10 @@ pub struct ListDirRequest { pub disk: ::prost::alloc::string::String, #[prost(string, tag = "2")] pub volume: ::prost::alloc::string::String, + #[prost(string, tag = "3")] + pub dir_path: ::prost::alloc::string::String, + #[prost(int32, tag = "4")] + pub count: i32, } #[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)] pub struct ListDirResponse { diff --git a/crates/protos/src/main.rs b/crates/protos/src/main.rs index 5184ed8a..fe18772a 100644 --- a/crates/protos/src/main.rs +++ b/crates/protos/src/main.rs @@ -16,7 +16,9 @@ use std::{cmp, env, fs, io::Write, path::Path, process::Command}; type AnyError = Box; +/// Expected version of `protoc` compiler. const VERSION_PROTOBUF: Version = Version(33, 1, 0); // 31.1.0 +/// Expected version of `flatc` compiler. const VERSION_FLATBUFFERS: Version = Version(25, 9, 23); // 25.9.23 /// Build protos if the major version of `flatc` or `protoc` is greater /// or lesser than the expected version. @@ -26,8 +28,10 @@ const ENV_FLATC_PATH: &str = "FLATC_PATH"; fn main() -> Result<(), AnyError> { let version = protobuf_compiler_version()?; + let need_compile = match version.compare_ext(&VERSION_PROTOBUF) { Ok(cmp::Ordering::Greater) => true, + Ok(cmp::Ordering::Equal) => true, Ok(_) => { if let Some(version_err) = Version::build_error_message(&version, &VERSION_PROTOBUF) { println!("cargo:warning=Tool `protoc` {version_err}, skip compiling."); @@ -42,6 +46,7 @@ fn main() -> Result<(), AnyError> { }; if !need_compile { + println!("no need to compile protos.{}", need_compile); return Ok(()); } @@ -121,13 +126,20 @@ fn main() -> Result<(), AnyError> { Err(_) => "flatc".to_string(), }; - compile_flatbuffers_models( + match compile_flatbuffers_models( &mut generated_mod_rs, &flatc_path, proto_dir.clone(), flatbuffer_out_dir.clone(), vec!["models"], - )?; + ) { + Ok(_) => { + println!("Successfully compiled flatbuffers models."); + } + Err(e) => { + return Err(format!("Failed to compile flatbuffers models: {e}").into()); + } + } fmt(); Ok(()) @@ -144,6 +156,7 @@ fn compile_flatbuffers_models, S: AsRef>( let version = flatbuffers_compiler_version(flatc_path)?; let need_compile = match version.compare_ext(&VERSION_FLATBUFFERS) { Ok(cmp::Ordering::Greater) => true, + Ok(cmp::Ordering::Equal) => true, Ok(_) => { if let Some(version_err) = Version::build_error_message(&version, &VERSION_FLATBUFFERS) { println!("cargo:warning=Tool `{flatc_path}` {version_err}, skip compiling."); @@ -161,6 +174,23 @@ fn compile_flatbuffers_models, S: AsRef>( // $rust_dir/mod.rs let mut sub_mod_rs = fs::File::create(rust_dir.join("mod.rs"))?; + writeln!( + &mut sub_mod_rs, + r#"// Copyright 2024 RustFS Team + // + // Licensed under the Apache License, Version 2.0 (the "License"); + // you may not use this file except in compliance with the License. + // You may obtain a copy of the License at + // + // http://www.apache.org/licenses/LICENSE-2.0 + // + // Unless required by applicable law or agreed to in writing, software + // distributed under the License is distributed on an "AS IS" BASIS, + // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + // See the License for the specific language governing permissions and + // limitations under the License."# + )?; + writeln!(&mut sub_mod_rs, "\n")?; writeln!(generated_mod_rs)?; writeln!(generated_mod_rs, "mod flatbuffers_generated;")?; for mod_name in mod_names.iter() { diff --git a/crates/protos/src/node.proto b/crates/protos/src/node.proto index 547e0e6b..c3b535c6 100644 --- a/crates/protos/src/node.proto +++ b/crates/protos/src/node.proto @@ -225,6 +225,8 @@ message ReadAtResponse { message ListDirRequest { string disk = 1; // indicate which one in the disks string volume = 2; + string dir_path = 3; + int32 count = 4; } message ListDirResponse { diff --git a/rustfs/src/storage/tonic_service.rs b/rustfs/src/storage/tonic_service.rs index bcee4903..eebe1c74 100644 --- a/rustfs/src/storage/tonic_service.rs +++ b/rustfs/src/storage/tonic_service.rs @@ -782,7 +782,7 @@ impl Node for NodeService { async fn list_dir(&self, request: Request) -> Result, Status> { let request = request.into_inner(); if let Some(disk) = self.find_disk(&request.disk).await { - match disk.list_dir("", &request.volume, "", 0).await { + match disk.list_dir("", &request.volume, &request.dir_path, request.count).await { Ok(volumes) => Ok(Response::new(ListDirResponse { success: true, volumes, @@ -2623,6 +2623,8 @@ mod tests { let request = Request::new(ListDirRequest { disk: "invalid-disk-path".to_string(), volume: "test-volume".to_string(), + dir_path: "test-dir-path".to_string(), + count: 10, }); let response = service.list_dir(request).await; diff --git a/scripts/install-flatc.sh b/scripts/install-flatc.sh new file mode 100755 index 00000000..1f95a9cc --- /dev/null +++ b/scripts/install-flatc.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Install flatc 25.9.23 on macOS + +set -e + +FLATC_VERSION="25.9.23" +ARCH=$(uname -m) +INSTALL_DIR="${HOME}/.local/bin" +FLATC_BIN="${INSTALL_DIR}/flatc" + +# Select download URL based on architecture +if [ "$ARCH" = "arm64" ]; then + # Apple Silicon (M1/M2/M3) + FLATC_URL="https://github.com/google/flatbuffers/releases/download/v${FLATC_VERSION}/Mac.flatc.binary.zip" +elif [ "$ARCH" = "x86_64" ]; then + # Intel Mac + FLATC_URL="https://github.com/google/flatbuffers/releases/download/v${FLATC_VERSION}/MacIntel.flatc.binary.zip" +else + echo "Error: Unsupported architecture $ARCH" + exit 1 +fi + +echo "Downloading flatc ${FLATC_VERSION} for ${ARCH}..." +TEMP_DIR=$(mktemp -d) +cd "$TEMP_DIR" + +# Download and extract +curl -L -o flatc.zip "$FLATC_URL" +unzip -q flatc.zip + +# Create install directory +mkdir -p "$INSTALL_DIR" + +# Backup existing version if present +if [ -f "$FLATC_BIN" ]; then + echo "Backing up existing flatc to ${FLATC_BIN}.backup" + mv "$FLATC_BIN" "${FLATC_BIN}.backup" +fi + +# Install flatc +cp flatc "$FLATC_BIN" +chmod +x "$FLATC_BIN" + +# Clean up temporary files +cd - +rm -rf "$TEMP_DIR" + +# Verify installation +echo "" +echo "Verifying installation..." +"$FLATC_BIN" --version + +# Check PATH +if [[ ":$PATH:" != *":${INSTALL_DIR}:"* ]]; then + echo "" + echo "⚠️ Warning: ${INSTALL_DIR} is not in PATH" + echo "Please add the following to ~/.zshrc or ~/.bash_profile:" + echo "" + echo " export PATH=\"\${HOME}/.local/bin:\$PATH\"" + echo "" + echo "Then run: source ~/.zshrc" +else + echo "" + echo "✅ flatc ${FLATC_VERSION} installed successfully!" + echo "Location: $FLATC_BIN" +fi + diff --git a/scripts/install-protoc.sh b/scripts/install-protoc.sh new file mode 100755 index 00000000..dfb52a0a --- /dev/null +++ b/scripts/install-protoc.sh @@ -0,0 +1,67 @@ +#!/bin/bash +# Install protoc 33.1 on macOS + +set -e + +PROTOC_VERSION="33.1" +ARCH=$(uname -m) +INSTALL_DIR="${HOME}/.local/bin" +PROTOC_BIN="${INSTALL_DIR}/protoc" + +# Select download URL based on architecture +if [ "$ARCH" = "arm64" ]; then + # Apple Silicon (M1/M2/M3) + PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-osx-aarch_64.zip" +elif [ "$ARCH" = "x86_64" ]; then + # Intel Mac + PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-osx-x86_64.zip" +else + echo "Error: Unsupported architecture $ARCH" + exit 1 +fi + +echo "Downloading protoc ${PROTOC_VERSION} for ${ARCH}..." +TEMP_DIR=$(mktemp -d) +cd "$TEMP_DIR" + +# Download and extract +curl -L -o protoc.zip "$PROTOC_URL" +unzip -q protoc.zip + +# Create install directory +mkdir -p "$INSTALL_DIR" + +# Backup existing version if present +if [ -f "$PROTOC_BIN" ]; then + echo "Backing up existing protoc to ${PROTOC_BIN}.backup" + mv "$PROTOC_BIN" "${PROTOC_BIN}.backup" +fi + +# Install protoc +cp bin/protoc "$PROTOC_BIN" +chmod +x "$PROTOC_BIN" + +# Clean up temporary files +cd - +rm -rf "$TEMP_DIR" + +# Verify installation +echo "" +echo "Verifying installation..." +"$PROTOC_BIN" --version + +# Check PATH +if [[ ":$PATH:" != *":${INSTALL_DIR}:"* ]]; then + echo "" + echo "⚠️ Warning: ${INSTALL_DIR} is not in PATH" + echo "Please add the following to ~/.zshrc or ~/.bash_profile:" + echo "" + echo " export PATH=\"\${HOME}/.local/bin:\$PATH\"" + echo "" + echo "Then run: source ~/.zshrc" +else + echo "" + echo "✅ protoc ${PROTOC_VERSION} installed successfully!" + echo "Location: $PROTOC_BIN" +fi +