diff --git a/.vscode/launch.json b/.vscode/launch.json index a7b272fa..8c61c183 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -19,6 +19,9 @@ "kind": "bin" } }, + "env": { + "RUST_LOG": "rustfs=debug,ecstore=info,s3s=debug" + }, "args": [ "--access-key", "AKEXAMPLERUSTFS", diff --git a/TODO.md b/TODO.md index f2a27b9f..aa3c2ba8 100644 --- a/TODO.md +++ b/TODO.md @@ -5,7 +5,7 @@ - [ ] 上传同名文件时,删除旧版本文件 - [ ] EC可用读写数量判断 - [ ] 小文件存储到metafile, inlinedata - - [ ] 错误类型判断 + - [ ] 错误类型判断,程序中判断错误类型,如何统一错误 - [ ] 优化并发执行 - [ ] 抽象出metafile存储 - [ ] 代码优化 @@ -13,14 +13,21 @@ ## 基础功能 - [ ] 桶操作 - - [x] 创建 - - [x] 列表 - - [x] 详情 + - [x] 创建 CreateBucket + - [x] 列表 ListBuckets + - [ ] 桶下面的文件列表 ListObjects + - [x] 详情 HeadBucket - [ ] 删除 - [ ] 文件操作 - - [x] 上传 - - [x] 大文件上传 - - [x] 下载 + - [x] 上传 PutObject + - [ ] 大文件上传 + - [ ] 创建分片上传 CreateMultipartUpload + - [x] 上传分片 PubObjectPart + - [x] 提交完成 CompleteMultipartUpload + - [ ] 取消上传 + - [x] 下载 GetObject + - [ ] 复制 CopyObject + - [ ] 详情 HeadObject - [ ] 删除 ## 扩展功能 diff --git a/ecstore/src/disk.rs b/ecstore/src/disk.rs index ad3ea699..7a933aed 100644 --- a/ecstore/src/disk.rs +++ b/ecstore/src/disk.rs @@ -1,6 +1,6 @@ use std::{ fs::Metadata, - io::SeekFrom, + io::{self, SeekFrom}, os::unix::ffi::OsStringExt, path::{Path, PathBuf}, sync::Arc, @@ -218,8 +218,14 @@ impl LocalDisk { if recursive { let trash_path = self.get_object_path(RUSTFS_META_TMP_DELETED_BUCKET, Uuid::new_v4().to_string().as_str())?; - fs::create_dir_all(&trash_path).await?; - fs::rename(&delete_path, &trash_path).await?; + // fs::create_dir_all(&trash_path).await?; + fs::rename(&delete_path, &trash_path).await.map_err(|err| { + // 使用文件路径自定义错误信息 + io::Error::new( + err.kind(), + format!("Failed to rename file '{:?}' to '{:?}': {}", &delete_path, &trash_path, err), + ) + })?; // TODO: immediate @@ -526,6 +532,40 @@ impl DiskAPI for LocalDisk { // Ok((buffer, bytes_read)) } + async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: usize) -> Result> { + let p = self.get_bucket_path(&volume)?; + + let mut entries = fs::read_dir(&p).await?; + + let mut volumes = Vec::new(); + + while let Some(entry) = entries.next_entry().await? { + if let Ok(metadata) = entry.metadata().await { + let vec = entry.file_name().into_vec(); + + // if !metadata.is_dir() { + // continue; + // } + + let name = match String::from_utf8(vec) { + Ok(s) => s, + Err(_) => return Err(Error::msg("Not supported utf8 file name on this platform")), + }; + + // let created = match metadata.created() { + // Ok(md) => OffsetDateTime::from(md), + // Err(_) => return Err(Error::msg("Not supported created on this platform")), + // }; + + volumes.push(name); + } + } + + Ok(volumes) + } + async fn walk_dir(&self) -> Result> { + unimplemented!() + } async fn rename_data( &self, src_volume: &str, @@ -648,9 +688,9 @@ impl DiskAPI for LocalDisk { if let Ok(metadata) = entry.metadata().await { let vec = entry.file_name().into_vec(); - if !metadata.is_dir() { - continue; - } + // if !metadata.is_dir() { + // continue; + // } let name = match String::from_utf8(vec) { Ok(s) => s, diff --git a/ecstore/src/disk_api.rs b/ecstore/src/disk_api.rs index 8f2c2316..f9bdf702 100644 --- a/ecstore/src/disk_api.rs +++ b/ecstore/src/disk_api.rs @@ -26,6 +26,10 @@ pub trait DiskAPI: Debug + Send + Sync + 'static { async fn create_file(&self, origvolume: &str, volume: &str, path: &str, file_size: usize) -> Result; async fn append_file(&self, volume: &str, path: &str) -> Result; async fn read_file(&self, volume: &str, path: &str) -> Result; + // 读目录下的所有文件、目录 + async fn list_dir(&self, origvolume: &str, volume: &str, dir_path: &str, count: usize) -> Result>; + // 读目录下的所有xl.meta + async fn walk_dir(&self) -> Result>; async fn rename_data( &self, src_volume: &str, @@ -206,8 +210,6 @@ pub enum DiskError { impl DiskError { pub fn check_disk_fatal_errs(errs: &Vec>) -> Result<()> { - println!("errs: {:?}", errs); - if Self::count_errs(errs, &DiskError::UnsupportedDisk) == errs.len() { return Err(Error::new(DiskError::UnsupportedDisk)); } diff --git a/ecstore/src/erasure.rs b/ecstore/src/erasure.rs index 66cd1979..f1a3c97d 100644 --- a/ecstore/src/erasure.rs +++ b/ecstore/src/erasure.rs @@ -29,6 +29,10 @@ pub struct Erasure { impl Erasure { pub fn new(data_shards: usize, parity_shards: usize, block_size: usize) -> Self { + warn!( + "Erasure new data_shards {},parity_shards {} block_size {} ", + data_shards, parity_shards, block_size + ); Erasure { data_shards, parity_shards, diff --git a/ecstore/src/store_init.rs b/ecstore/src/store_init.rs index fdcf2565..0190c6a1 100644 --- a/ecstore/src/store_init.rs +++ b/ecstore/src/store_init.rs @@ -1,4 +1,5 @@ use futures::future::join_all; +use tracing::warn; use uuid::Uuid; use crate::{ @@ -99,6 +100,7 @@ fn get_format_file_in_quorum(formats: &Vec>) -> Result