sdgoij/rustfs
1
0
Fork 0
mirror of https://github.com/rustfs/rustfs.git synced 2026-01-16 17:20:33 +00:00
Code Issues Packages Projects Releases Wiki Activity

fix: listdir rpc (#979)

Browse Source 
Co-authored-by: houseme <housemecn@gmail.com>
Co-authored-by: loverustfs <hello@rustfs.com>
This commit is contained in:
weisd
2025-12-04 16:12:10 +08:00
committed by GitHub
parent 3fd003b21d
commit b3c80ae362
11 changed files with 210 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
Cargo.lock generated
View File

@@ -691,9 +691,9 @@ dependencies = [
[[package]]
name = "aws-sdk-s3"
version = "1.115.0"
version = "1.116.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fdaa0053cbcbc384443dd24569bd5d1664f86427b9dc04677bd0ab853954baec"
checksum = "cd4c10050aa905b50dc2a1165a9848d598a80c3a724d6f93b5881aa62235e4a5"
dependencies = [
"aws-credential-types",
"aws-runtime",
@@ -4266,9 +4266,9 @@ dependencies = [
[[package]]
name = "hyper-util"
version = "0.1.18"
version = "0.1.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56"
checksum = "727805d60e7938b76b826a6ef209eb70eaa1812794f9424d4a4e2d740662df5f"
dependencies = [
"base64 0.22.1",
"bytes",
@@ -4816,9 +4816,9 @@ dependencies = [
[[package]]
name = "libz-rs-sys"
version = "0.5.2"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd"
checksum = "8b484ba8d4f775eeca644c452a56650e544bf7e617f1d170fe7298122ead5222"
dependencies = [
"zlib-rs",
]
@@ -4875,11 +4875,11 @@ dependencies = [
[[package]]
name = "log"
version = "0.4.28"
version = "0.4.29"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"
checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897"
dependencies = [
"serde",
"serde_core",
"value-bag",
]
@@ -10394,9 +10394,9 @@ dependencies = [
[[package]]
name = "zlib-rs"
version = "0.5.2"
version = "0.5.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2"
checksum = "36134c44663532e6519d7a6dfdbbe06f6f8192bde8ae9ed076e9b213f0e31df7"
[[package]]
name = "zopfli"

4
Cargo.toml
View File

@@ -105,7 +105,7 @@ futures-core = "0.3.31"
futures-util = "0.3.31"
hyper = { version = "1.8.1", features = ["http2", "http1", "server"] }
hyper-rustls = { version = "0.27.7", default-features = false, features = ["native-tokio", "http1", "tls12", "logging", "http2", "ring", "webpki-roots"] }
hyper-util = { version = "0.1.18", features = ["tokio", "server-auto", "server-graceful"] }
hyper-util = { version = "0.1.19", features = ["tokio", "server-auto", "server-graceful"] }
http = "1.4.0"
http-body = "1.0.1"
reqwest = { version = "0.12.24", default-features = false, features = ["rustls-tls-webpki-roots", "charset", "http2", "system-proxy", "stream", "json", "blocking"] }
@@ -167,7 +167,7 @@ atoi = "2.0.0"
atomic_enum = "0.3.0"
aws-config = { version = "1.8.11" }
aws-credential-types = { version = "1.2.10" }
aws-sdk-s3 = { version = "1.115.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] }
aws-sdk-s3 = { version = "1.116.0", default-features = false, features = ["sigv4a", "rustls", "rt-tokio"] }
aws-smithy-types = { version = "1.3.4" }
base64 = "0.22.1"
base64-simd = "0.8.0"

6
crates/ecstore/src/disk/mod.rs
View File

@@ -271,10 +271,10 @@ impl DiskAPI for Disk {
}
#[tracing::instrument(skip(self))]
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
async fn list_dir(&self, _origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
match self {
Disk::Local(local_disk) => local_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
Disk::Remote(remote_disk) => remote_disk.list_dir(_origvolume, volume, _dir_path, _count).await,
Disk::Local(local_disk) => local_disk.list_dir(_origvolume, volume, dir_path, count).await,
Disk::Remote(remote_disk) => remote_disk.list_dir(_origvolume, volume, dir_path, count).await,
}
}

8
crates/ecstore/src/rpc/remote_disk.rs
View File

@@ -42,7 +42,7 @@ use rustfs_protos::proto_gen::node_service::RenamePartRequest;
use rustfs_rio::{HttpReader, HttpWriter};
use tokio::{io::AsyncWrite, net::TcpStream, time::timeout};
use tonic::Request;
use tracing::info;
use tracing::{debug, info};
use uuid::Uuid;
#[derive(Debug)]
@@ -596,14 +596,16 @@ impl DiskAPI for RemoteDisk {
}
#[tracing::instrument(skip(self))]
async fn list_dir(&self, _origvolume: &str, volume: &str, _dir_path: &str, _count: i32) -> Result<Vec<String>> {
info!("list_dir {}/{}", volume, _dir_path);
async fn list_dir(&self, _origvolume: &str, volume: &str, dir_path: &str, count: i32) -> Result<Vec<String>> {
debug!("list_dir {}/{}", volume, dir_path);
let mut client = node_service_time_out_client(&self.addr)
.await
.map_err(|err| Error::other(format!("can not get client, err: {err}")))?;
let request = Request::new(ListDirRequest {
disk: self.endpoint.to_string(),
volume: volume.to_string(),
dir_path: dir_path.to_string(),
count,
});
let response = client.list_dir(request).await?.into_inner();

14
crates/protos/src/generated/flatbuffers_generated/mod.rs
View File

@@ -1 +1,15 @@
// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
pub mod models;

4
crates/protos/src/generated/proto_gen/node_service.rs
View File

@@ -303,6 +303,10 @@ pub struct ListDirRequest {
pub disk: ::prost::alloc::string::String,
#[prost(string, tag = "2")]
pub volume: ::prost::alloc::string::String,
#[prost(string, tag = "3")]
pub dir_path: ::prost::alloc::string::String,
#[prost(int32, tag = "4")]
pub count: i32,
}
#[derive(Clone, PartialEq, Eq, Hash, ::prost::Message)]
pub struct ListDirResponse {

34
crates/protos/src/main.rs
View File

@@ -16,7 +16,9 @@ use std::{cmp, env, fs, io::Write, path::Path, process::Command};
type AnyError = Box<dyn std::error::Error>;
/// Expected version of `protoc` compiler.
const VERSION_PROTOBUF: Version = Version(33, 1, 0); // 31.1.0
/// Expected version of `flatc` compiler.
const VERSION_FLATBUFFERS: Version = Version(25, 9, 23); // 25.9.23
/// Build protos if the major version of `flatc` or `protoc` is greater
/// or lesser than the expected version.
@@ -26,8 +28,10 @@ const ENV_FLATC_PATH: &str = "FLATC_PATH";
fn main() -> Result<(), AnyError> {
let version = protobuf_compiler_version()?;
let need_compile = match version.compare_ext(&VERSION_PROTOBUF) {
Ok(cmp::Ordering::Greater) => true,
Ok(cmp::Ordering::Equal) => true,
Ok(_) => {
if let Some(version_err) = Version::build_error_message(&version, &VERSION_PROTOBUF) {
println!("cargo:warning=Tool `protoc` {version_err}, skip compiling.");
@@ -42,6 +46,7 @@ fn main() -> Result<(), AnyError> {
};
if !need_compile {
println!("no need to compile protos.{}", need_compile);
return Ok(());
}
@@ -121,13 +126,20 @@ fn main() -> Result<(), AnyError> {
Err(_) => "flatc".to_string(),
};
compile_flatbuffers_models(
match compile_flatbuffers_models(
&mut generated_mod_rs,
&flatc_path,
proto_dir.clone(),
flatbuffer_out_dir.clone(),
vec!["models"],
)?;
) {
Ok(_) => {
println!("Successfully compiled flatbuffers models.");
}
Err(e) => {
return Err(format!("Failed to compile flatbuffers models: {e}").into());
}
}
fmt();
Ok(())
@@ -144,6 +156,7 @@ fn compile_flatbuffers_models<P: AsRef<Path>, S: AsRef<str>>(
let version = flatbuffers_compiler_version(flatc_path)?;
let need_compile = match version.compare_ext(&VERSION_FLATBUFFERS) {
Ok(cmp::Ordering::Greater) => true,
Ok(cmp::Ordering::Equal) => true,
Ok(_) => {
if let Some(version_err) = Version::build_error_message(&version, &VERSION_FLATBUFFERS) {
println!("cargo:warning=Tool `{flatc_path}` {version_err}, skip compiling.");
@@ -161,6 +174,23 @@ fn compile_flatbuffers_models<P: AsRef<Path>, S: AsRef<str>>(
// $rust_dir/mod.rs
let mut sub_mod_rs = fs::File::create(rust_dir.join("mod.rs"))?;
writeln!(
&mut sub_mod_rs,
r#"// Copyright 2024 RustFS Team
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License."#
)?;
writeln!(&mut sub_mod_rs, "\n")?;
writeln!(generated_mod_rs)?;
writeln!(generated_mod_rs, "mod flatbuffers_generated;")?;
for mod_name in mod_names.iter() {

2
crates/protos/src/node.proto
View File

@@ -225,6 +225,8 @@ message ReadAtResponse {
message ListDirRequest {
string disk = 1; // indicate which one in the disks
string volume = 2;
string dir_path = 3;
int32 count = 4;
}
message ListDirResponse {

4
rustfs/src/storage/tonic_service.rs
View File

@@ -782,7 +782,7 @@ impl Node for NodeService {
async fn list_dir(&self, request: Request<ListDirRequest>) -> Result<Response<ListDirResponse>, Status> {
let request = request.into_inner();
if let Some(disk) = self.find_disk(&request.disk).await {
match disk.list_dir("", &request.volume, "", 0).await {
match disk.list_dir("", &request.volume, &request.dir_path, request.count).await {
Ok(volumes) => Ok(Response::new(ListDirResponse {
success: true,
volumes,
@@ -2623,6 +2623,8 @@ mod tests {
let request = Request::new(ListDirRequest {
disk: "invalid-disk-path".to_string(),
volume: "test-volume".to_string(),
dir_path: "test-dir-path".to_string(),
count: 10,
});
let response = service.list_dir(request).await;

67
scripts/install-flatc.sh Executable file
View File

@@ -0,0 +1,67 @@
#!/bin/bash
# Install flatc 25.9.23 on macOS
set -e
FLATC_VERSION="25.9.23"
ARCH=$(uname -m)
INSTALL_DIR="${HOME}/.local/bin"
FLATC_BIN="${INSTALL_DIR}/flatc"
# Select download URL based on architecture
if [ "$ARCH" = "arm64" ]; then
# Apple Silicon (M1/M2/M3)
FLATC_URL="https://github.com/google/flatbuffers/releases/download/v${FLATC_VERSION}/Mac.flatc.binary.zip"
elif [ "$ARCH" = "x86_64" ]; then
# Intel Mac
FLATC_URL="https://github.com/google/flatbuffers/releases/download/v${FLATC_VERSION}/MacIntel.flatc.binary.zip"
else
echo "Error: Unsupported architecture $ARCH"
exit 1
fi
echo "Downloading flatc ${FLATC_VERSION} for ${ARCH}..."
TEMP_DIR=$(mktemp -d)
cd "$TEMP_DIR"
# Download and extract
curl -L -o flatc.zip "$FLATC_URL"
unzip -q flatc.zip
# Create install directory
mkdir -p "$INSTALL_DIR"
# Backup existing version if present
if [ -f "$FLATC_BIN" ]; then
echo "Backing up existing flatc to ${FLATC_BIN}.backup"
mv "$FLATC_BIN" "${FLATC_BIN}.backup"
fi
# Install flatc
cp flatc "$FLATC_BIN"
chmod +x "$FLATC_BIN"
# Clean up temporary files
cd -
rm -rf "$TEMP_DIR"
# Verify installation
echo ""
echo "Verifying installation..."
"$FLATC_BIN" --version
# Check PATH
if [[ ":$PATH:" != *":${INSTALL_DIR}:"* ]]; then
echo ""
echo "⚠️ Warning: ${INSTALL_DIR} is not in PATH"
echo "Please add the following to ~/.zshrc or ~/.bash_profile:"
echo ""
echo " export PATH=\"\${HOME}/.local/bin:\$PATH\""
echo ""
echo "Then run: source ~/.zshrc"
else
echo ""
echo "✅ flatc ${FLATC_VERSION} installed successfully!"
echo "Location: $FLATC_BIN"
fi

67
scripts/install-protoc.sh Executable file
View File

@@ -0,0 +1,67 @@
#!/bin/bash
# Install protoc 33.1 on macOS
set -e
PROTOC_VERSION="33.1"
ARCH=$(uname -m)
INSTALL_DIR="${HOME}/.local/bin"
PROTOC_BIN="${INSTALL_DIR}/protoc"
# Select download URL based on architecture
if [ "$ARCH" = "arm64" ]; then
# Apple Silicon (M1/M2/M3)
PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-osx-aarch_64.zip"
elif [ "$ARCH" = "x86_64" ]; then
# Intel Mac
PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-osx-x86_64.zip"
else
echo "Error: Unsupported architecture $ARCH"
exit 1
fi
echo "Downloading protoc ${PROTOC_VERSION} for ${ARCH}..."
TEMP_DIR=$(mktemp -d)
cd "$TEMP_DIR"
# Download and extract
curl -L -o protoc.zip "$PROTOC_URL"
unzip -q protoc.zip
# Create install directory
mkdir -p "$INSTALL_DIR"
# Backup existing version if present
if [ -f "$PROTOC_BIN" ]; then
echo "Backing up existing protoc to ${PROTOC_BIN}.backup"
mv "$PROTOC_BIN" "${PROTOC_BIN}.backup"
fi
# Install protoc
cp bin/protoc "$PROTOC_BIN"
chmod +x "$PROTOC_BIN"
# Clean up temporary files
cd -
rm -rf "$TEMP_DIR"
# Verify installation
echo ""
echo "Verifying installation..."
"$PROTOC_BIN" --version
# Check PATH
if [[ ":$PATH:" != *":${INSTALL_DIR}:"* ]]; then
echo ""
echo "⚠️ Warning: ${INSTALL_DIR} is not in PATH"
echo "Please add the following to ~/.zshrc or ~/.bash_profile:"
echo ""
echo " export PATH=\"\${HOME}/.local/bin:\$PATH\""
echo ""
echo "Then run: source ~/.zshrc"
else
echo ""
echo "✅ protoc ${PROTOC_VERSION} installed successfully!"
echo "Location: $PROTOC_BIN"
fi