From 2eeb9dbcbce71fad63b142dd7f291be638fddd76 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 11 Jun 2025 00:30:10 +0800 Subject: [PATCH] fix cargo test error, delete unnecessary files --- crates/rio/src/bitrot.rs | 331 --- ecstore/src/bitrot.rs | 3 +- ecstore/src/disk/local.rs | 7 +- ecstore/src/erasure_coding/bitrot.rs | 2 + ecstore/src/file_meta.rs | 3384 -------------------------- ecstore/src/file_meta_inline.rs | 238 -- ecstore/src/io.rs | 580 ----- ecstore/src/metacache/mod.rs | 1 - ecstore/src/metacache/writer.rs | 387 --- ecstore/src/metrics_realtime.rs | 3 +- ecstore/src/quorum.rs | 268 -- ecstore/src/utils/mod.rs | 2 +- ecstore/src/utils/os/linux.rs | 178 -- ecstore/src/utils/os/mod.rs | 338 --- ecstore/src/utils/os/unix.rs | 73 - ecstore/src/utils/os/windows.rs | 144 -- 16 files changed, 11 insertions(+), 5928 deletions(-) delete mode 100644 crates/rio/src/bitrot.rs delete mode 100644 ecstore/src/file_meta.rs delete mode 100644 ecstore/src/file_meta_inline.rs delete mode 100644 ecstore/src/io.rs delete mode 100644 ecstore/src/metacache/mod.rs delete mode 100644 ecstore/src/metacache/writer.rs delete mode 100644 ecstore/src/quorum.rs delete mode 100644 ecstore/src/utils/os/linux.rs delete mode 100644 ecstore/src/utils/os/mod.rs delete mode 100644 ecstore/src/utils/os/unix.rs delete mode 100644 ecstore/src/utils/os/windows.rs diff --git a/crates/rio/src/bitrot.rs b/crates/rio/src/bitrot.rs deleted file mode 100644 index 76a4e97e..00000000 --- a/crates/rio/src/bitrot.rs +++ /dev/null @@ -1,331 +0,0 @@ -use pin_project_lite::pin_project; -use rustfs_utils::{HashAlgorithm, read_full, write_all}; -use tokio::io::{AsyncRead, AsyncReadExt, AsyncWrite}; - -pin_project! { - /// BitrotReader reads (hash+data) blocks from an async reader and verifies hash integrity. - pub struct BitrotReader { - #[pin] - inner: R, - hash_algo: HashAlgorithm, - shard_size: usize, - buf: Vec, - hash_buf: Vec, - hash_read: usize, - data_buf: Vec, - data_read: usize, - hash_checked: bool, - } -} - -impl BitrotReader -where - R: AsyncRead + Unpin + Send + Sync, -{ - /// Create a new BitrotReader. - pub fn new(inner: R, shard_size: usize, algo: HashAlgorithm) -> Self { - let hash_size = algo.size(); - Self { - inner, - hash_algo: algo, - shard_size, - buf: Vec::new(), - hash_buf: vec![0u8; hash_size], - hash_read: 0, - data_buf: Vec::new(), - data_read: 0, - hash_checked: false, - } - } - - /// Read a single (hash+data) block, verify hash, and return the number of bytes read into `out`. - /// Returns an error if hash verification fails or data exceeds shard_size. - pub async fn read(&mut self, out: &mut [u8]) -> std::io::Result { - if out.len() > self.shard_size { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!("data size {} exceeds shard size {}", out.len(), self.shard_size), - )); - } - - let hash_size = self.hash_algo.size(); - // Read hash - let mut hash_buf = vec![0u8; hash_size]; - if hash_size > 0 { - self.inner.read_exact(&mut hash_buf).await?; - } - - let data_len = read_full(&mut self.inner, out).await?; - - // // Read data - // let mut data_len = 0; - // while data_len < out.len() { - // let n = self.inner.read(&mut out[data_len..]).await?; - // if n == 0 { - // break; - // } - // data_len += n; - // // Only read up to one shard_size block - // if data_len >= self.shard_size { - // break; - // } - // } - - if hash_size > 0 { - let actual_hash = self.hash_algo.hash_encode(&out[..data_len]); - if actual_hash != hash_buf { - return Err(std::io::Error::new(std::io::ErrorKind::InvalidData, "bitrot hash mismatch")); - } - } - Ok(data_len) - } -} - -pin_project! { - /// BitrotWriter writes (hash+data) blocks to an async writer. - pub struct BitrotWriter { - #[pin] - inner: W, - hash_algo: HashAlgorithm, - shard_size: usize, - buf: Vec, - finished: bool, - } -} - -impl BitrotWriter -where - W: AsyncWrite + Unpin + Send + Sync, -{ - /// Create a new BitrotWriter. - pub fn new(inner: W, shard_size: usize, algo: HashAlgorithm) -> Self { - let hash_algo = algo; - Self { - inner, - hash_algo, - shard_size, - buf: Vec::new(), - finished: false, - } - } - - pub fn into_inner(self) -> W { - self.inner - } - - /// Write a (hash+data) block. Returns the number of data bytes written. - /// Returns an error if called after a short write or if data exceeds shard_size. - pub async fn write(&mut self, buf: &[u8]) -> std::io::Result { - if buf.is_empty() { - return Ok(0); - } - - if self.finished { - return Err(std::io::Error::new(std::io::ErrorKind::InvalidInput, "bitrot writer already finished")); - } - - if buf.len() > self.shard_size { - return Err(std::io::Error::new( - std::io::ErrorKind::InvalidInput, - format!("data size {} exceeds shard size {}", buf.len(), self.shard_size), - )); - } - - if buf.len() < self.shard_size { - self.finished = true; - } - - let hash_algo = &self.hash_algo; - - if hash_algo.size() > 0 { - let hash = hash_algo.hash_encode(buf); - self.buf.extend_from_slice(&hash); - } - - self.buf.extend_from_slice(buf); - - // Write hash+data in one call - let mut n = write_all(&mut self.inner, &self.buf).await?; - - if n < hash_algo.size() { - return Err(std::io::Error::new( - std::io::ErrorKind::WriteZero, - "short write: not enough bytes written", - )); - } - - n -= hash_algo.size(); - - self.buf.clear(); - - Ok(n) - } -} - -pub fn bitrot_shard_file_size(size: usize, shard_size: usize, algo: HashAlgorithm) -> usize { - if algo != HashAlgorithm::HighwayHash256S { - return size; - } - size.div_ceil(shard_size) * algo.size() + size -} - -pub async fn bitrot_verify( - mut r: R, - want_size: usize, - part_size: usize, - algo: HashAlgorithm, - _want: Vec, - mut shard_size: usize, -) -> std::io::Result<()> { - let mut hash_buf = vec![0; algo.size()]; - let mut left = want_size; - - if left != bitrot_shard_file_size(part_size, shard_size, algo.clone()) { - return Err(std::io::Error::other("bitrot shard file size mismatch")); - } - - while left > 0 { - let n = r.read_exact(&mut hash_buf).await?; - left -= n; - - if left < shard_size { - shard_size = left; - } - - let mut buf = vec![0; shard_size]; - let read = r.read_exact(&mut buf).await?; - - let actual_hash = algo.hash_encode(&buf); - if actual_hash != hash_buf[0..n] { - return Err(std::io::Error::other("bitrot hash mismatch")); - } - - left -= read; - } - - Ok(()) -} - -#[cfg(test)] -mod tests { - - use crate::{BitrotReader, BitrotWriter}; - use rustfs_utils::HashAlgorithm; - use std::io::Cursor; - - #[tokio::test] - async fn test_bitrot_read_write_ok() { - let data = b"hello world! this is a test shard."; - let data_size = data.len(); - let shard_size = 8; - - let buf: Vec = Vec::new(); - let writer = Cursor::new(buf); - let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256); - - let mut n = 0; - for chunk in data.chunks(shard_size) { - n += bitrot_writer.write(chunk).await.unwrap(); - } - assert_eq!(n, data.len()); - - // 读 - let reader = bitrot_writer.into_inner(); - let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256); - let mut out = Vec::new(); - let mut n = 0; - while n < data_size { - let mut buf = vec![0u8; shard_size]; - let m = bitrot_reader.read(&mut buf).await.unwrap(); - if m == 0 { - break; - } - assert_eq!(&buf[..m], &data[n..n + m]); - - out.extend_from_slice(&buf[..m]); - n += m; - } - - assert_eq!(n, data_size); - assert_eq!(data, &out[..]); - } - - #[tokio::test] - async fn test_bitrot_read_hash_mismatch() { - let data = b"test data for bitrot"; - let data_size = data.len(); - let shard_size = 8; - let buf: Vec = Vec::new(); - let writer = Cursor::new(buf); - let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::HighwayHash256); - for chunk in data.chunks(shard_size) { - let _ = bitrot_writer.write(chunk).await.unwrap(); - } - let mut written = bitrot_writer.into_inner().into_inner(); - // change the last byte to make hash mismatch - let pos = written.len() - 1; - written[pos] ^= 0xFF; - let reader = Cursor::new(written); - let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256); - - let count = data_size.div_ceil(shard_size); - - let mut idx = 0; - let mut n = 0; - while n < data_size { - let mut buf = vec![0u8; shard_size]; - let res = bitrot_reader.read(&mut buf).await; - - if idx == count - 1 { - // 最后一个块，应该返回错误 - assert!(res.is_err()); - assert_eq!(res.unwrap_err().kind(), std::io::ErrorKind::InvalidData); - break; - } - - let m = res.unwrap(); - if m == 0 { - break; - } - - assert_eq!(&buf[..m], &data[n..n + m]); - - n += m; - idx += 1; - } - } - - #[tokio::test] - async fn test_bitrot_read_write_none_hash() { - let data = b"bitrot none hash test data!"; - let data_size = data.len(); - let shard_size = 8; - - let buf: Vec = Vec::new(); - let writer = Cursor::new(buf); - let mut bitrot_writer = BitrotWriter::new(writer, shard_size, HashAlgorithm::None); - - let mut n = 0; - for chunk in data.chunks(shard_size) { - n += bitrot_writer.write(chunk).await.unwrap(); - } - assert_eq!(n, data.len()); - - let reader = bitrot_writer.into_inner(); - let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::None); - let mut out = Vec::new(); - let mut n = 0; - while n < data_size { - let mut buf = vec![0u8; shard_size]; - let m = bitrot_reader.read(&mut buf).await.unwrap(); - if m == 0 { - break; - } - assert_eq!(&buf[..m], &data[n..n + m]); - out.extend_from_slice(&buf[..m]); - n += m; - } - assert_eq!(n, data_size); - assert_eq!(data, &out[..]); - } -} diff --git a/ecstore/src/bitrot.rs b/ecstore/src/bitrot.rs index 2224de82..fa2f5922 100644 --- a/ecstore/src/bitrot.rs +++ b/ecstore/src/bitrot.rs @@ -162,6 +162,7 @@ mod tests { assert!(wrapper.is_err()); let error = wrapper.unwrap_err(); - assert!(error.to_string().contains("io error")); + println!("error: {:?}", error); + assert_eq!(error, DiskError::DiskNotFound); } } diff --git a/ecstore/src/disk/local.rs b/ecstore/src/disk/local.rs index 1985e191..b89ed4ca 100644 --- a/ecstore/src/disk/local.rs +++ b/ecstore/src/disk/local.rs @@ -2,7 +2,7 @@ use super::error::{Error, Result}; use super::os::{is_root_disk, rename_all}; use super::{ BUCKET_META_PREFIX, CheckPartsResp, DeleteOptions, DiskAPI, DiskInfo, DiskInfoOptions, DiskLocation, DiskMetrics, - FileInfoVersions, Info, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, + FileInfoVersions, RUSTFS_META_BUCKET, ReadMultipleReq, ReadMultipleResp, ReadOptions, RenameDataResp, STORAGE_FORMAT_FILE_BACKUP, UpdateMetadataOpts, VolumeInfo, WalkDirOptions, os, }; use super::{endpoint::Endpoint, error::DiskError, format::FormatV3}; @@ -32,7 +32,7 @@ use crate::heal::heal_commands::{HealScanMode, HealingTracker}; use crate::heal::heal_ops::HEALING_TRACKER_FILENAME; use crate::new_object_layer_fn; use crate::store_api::{ObjectInfo, StorageAPI}; -use crate::utils::os::get_info; +// use crate::utils::os::get_info; use crate::utils::path::{ GLOBAL_DIR_SUFFIX, GLOBAL_DIR_SUFFIX_WITH_SLASH, SLASH_SEPARATOR, clean, decode_dir_object, encode_dir_object, has_suffix, path_join, path_join_buf, @@ -46,6 +46,7 @@ use rustfs_filemeta::{ read_xl_meta_no_data, }; use rustfs_utils::HashAlgorithm; +use rustfs_utils::os::get_info; use std::collections::{HashMap, HashSet}; use std::fmt::Debug; use std::io::SeekFrom; @@ -2284,7 +2285,7 @@ impl DiskAPI for LocalDisk { } } -async fn get_disk_info(drive_path: PathBuf) -> Result<(Info, bool)> { +async fn get_disk_info(drive_path: PathBuf) -> Result<(rustfs_utils::os::DiskInfo, bool)> { let drive_path = drive_path.to_string_lossy().to_string(); check_path_length(&drive_path)?; diff --git a/ecstore/src/erasure_coding/bitrot.rs b/ecstore/src/erasure_coding/bitrot.rs index 8bc375c3..a53165d7 100644 --- a/ecstore/src/erasure_coding/bitrot.rs +++ b/ecstore/src/erasure_coding/bitrot.rs @@ -367,6 +367,7 @@ mod tests { // 读 let reader = bitrot_writer.into_inner(); + let reader = Cursor::new(reader.into_inner()); let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::HighwayHash256); let mut out = Vec::new(); let mut n = 0; @@ -442,6 +443,7 @@ mod tests { assert_eq!(n, data.len()); let reader = bitrot_writer.into_inner(); + let reader = Cursor::new(reader.into_inner()); let mut bitrot_reader = BitrotReader::new(reader, shard_size, HashAlgorithm::None); let mut out = Vec::new(); let mut n = 0; diff --git a/ecstore/src/file_meta.rs b/ecstore/src/file_meta.rs deleted file mode 100644 index 3a87d3e1..00000000 --- a/ecstore/src/file_meta.rs +++ /dev/null @@ -1,3384 +0,0 @@ -use crate::disk::FileInfoVersions; -use crate::error::StorageError; -use crate::file_meta_inline::InlineData; -use crate::store_api::RawFileInfo; -use crate::{ - disk::error::DiskError, - store_api::{ERASURE_ALGORITHM, ErasureInfo, FileInfo, ObjectPartInfo}, -}; -use byteorder::ByteOrder; -use common::error::{Error, Result}; -use rmp::Marker; -use serde::{Deserialize, Serialize}; -use std::cmp::Ordering; -use std::fmt::Display; -use std::io::{self, Read, Write}; -use std::{collections::HashMap, io::Cursor}; -use time::OffsetDateTime; -use tokio::io::AsyncRead; -use tracing::{error, warn}; -use uuid::Uuid; -use xxhash_rust::xxh64; - -// XL header specifies the format -pub static XL_FILE_HEADER: [u8; 4] = [b'X', b'L', b'2', b' ']; -// pub static XL_FILE_VERSION_CURRENT: [u8; 4] = [0; 4]; - -// Current version being written. -// static XL_FILE_VERSION: [u8; 4] = [1, 0, 3, 0]; -static XL_FILE_VERSION_MAJOR: u16 = 1; -static XL_FILE_VERSION_MINOR: u16 = 3; -static XL_HEADER_VERSION: u8 = 3; -static XL_META_VERSION: u8 = 2; -static XXHASH_SEED: u64 = 0; - -const XL_FLAG_FREE_VERSION: u8 = 1 << 0; -// const XL_FLAG_USES_DATA_DIR: u8 = 1 << 1; -const _XL_FLAG_INLINE_DATA: u8 = 1 << 2; - -const META_DATA_READ_DEFAULT: usize = 4 << 10; -const MSGP_UINT32_SIZE: usize = 5; - -// type ScanHeaderVersionFn = Box Result<()>>; - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct FileMeta { - pub versions: Vec, - pub data: InlineData, // TODO: xlMetaInlineData - pub meta_ver: u8, -} - -impl FileMeta { - pub fn new() -> Self { - Self { - meta_ver: XL_META_VERSION, - data: InlineData::new(), - ..Default::default() - } - } - - // isXL2V1Format - #[tracing::instrument(level = "debug", skip_all)] - pub fn is_xl2_v1_format(buf: &[u8]) -> bool { - !matches!(Self::check_xl2_v1(buf), Err(_e)) - } - - #[tracing::instrument(level = "debug", skip_all)] - pub fn load(buf: &[u8]) -> Result { - let mut xl = FileMeta::default(); - xl.unmarshal_msg(buf)?; - - Ok(xl) - } - - // check_xl2_v1 读 xl 文件头，返回后续内容，版本信息 - // checkXL2V1 - #[tracing::instrument(level = "debug", skip_all)] - pub fn check_xl2_v1(buf: &[u8]) -> Result<(&[u8], u16, u16)> { - if buf.len() < 8 { - return Err(Error::msg("xl file header not exists")); - } - - if buf[0..4] != XL_FILE_HEADER { - return Err(Error::msg("xl file header err")); - } - - let major = byteorder::LittleEndian::read_u16(&buf[4..6]); - let minor = byteorder::LittleEndian::read_u16(&buf[6..8]); - if major > XL_FILE_VERSION_MAJOR { - return Err(Error::msg("xl file version err")); - } - - Ok((&buf[8..], major, minor)) - } - - // 固定 u32 - pub fn read_bytes_header(buf: &[u8]) -> Result<(u32, &[u8])> { - if buf.len() < 5 { - return Err(Error::new(io::Error::new( - io::ErrorKind::UnexpectedEof, - format!("Buffer too small: {} bytes, need at least 5", buf.len()), - ))); - } - - let (mut size_buf, _) = buf.split_at(5); - - // 取 meta 数据，buf = crc + data - let bin_len = rmp::decode::read_bin_len(&mut size_buf)?; - - Ok((bin_len, &buf[5..])) - } - - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let i = buf.len() as u64; - - // check version, buf = buf[8..] - let (buf, _, _) = Self::check_xl2_v1(buf)?; - - let (mut size_buf, buf) = buf.split_at(5); - - // 取 meta 数据，buf = crc + data - let bin_len = rmp::decode::read_bin_len(&mut size_buf)?; - - let (meta, buf) = buf.split_at(bin_len as usize); - - let (mut crc_buf, buf) = buf.split_at(5); - - // crc check - let crc = rmp::decode::read_u32(&mut crc_buf)?; - let meta_crc = xxh64::xxh64(meta, XXHASH_SEED) as u32; - - if crc != meta_crc { - return Err(Error::msg("xl file crc check failed")); - } - - if !buf.is_empty() { - self.data.update(buf); - self.data.validate()?; - } - - // 解析 meta - if !meta.is_empty() { - let (versions_len, _, meta_ver, meta) = Self::decode_xl_headers(meta)?; - - // let (_, meta) = meta.split_at(read_size as usize); - - self.meta_ver = meta_ver; - - self.versions = Vec::with_capacity(versions_len); - - let mut cur: Cursor<&[u8]> = Cursor::new(meta); - for _ in 0..versions_len { - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - let header_buf = &meta[start..end]; - - let mut ver = FileMetaShallowVersion::default(); - ver.header.unmarshal_msg(header_buf)?; - - cur.set_position(end as u64); - - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - let mut ver_meta_buf = &meta[start..end]; - - ver_meta_buf.read_to_end(&mut ver.meta)?; - - cur.set_position(end as u64); - - self.versions.push(ver); - } - } - - Ok(i) - } - - // decode_xl_headers 解析 meta 头，返回 (versions 数量，xl_header_version, xl_meta_version, 已读数据长度) - #[tracing::instrument(level = "debug", skip_all)] - fn decode_xl_headers(buf: &[u8]) -> Result<(usize, u8, u8, &[u8])> { - let mut cur = Cursor::new(buf); - - let header_ver: u8 = rmp::decode::read_int(&mut cur)?; - - if header_ver > XL_HEADER_VERSION { - return Err(Error::msg("xl header version invalid")); - } - - let meta_ver: u8 = rmp::decode::read_int(&mut cur)?; - if meta_ver > XL_META_VERSION { - return Err(Error::msg("xl meta version invalid")); - } - - let versions_len: usize = rmp::decode::read_int(&mut cur)?; - - Ok((versions_len, header_ver, meta_ver, &buf[cur.position() as usize..])) - } - - fn decode_versions Result<()>>(buf: &[u8], versions: usize, mut fnc: F) -> Result<()> { - let mut cur: Cursor<&[u8]> = Cursor::new(buf); - - for i in 0..versions { - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - let header_buf = &buf[start..end]; - - cur.set_position(end as u64); - - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - let ver_meta_buf = &buf[start..end]; - - cur.set_position(end as u64); - - if let Err(err) = fnc(i, header_buf, ver_meta_buf) { - if let Some(e) = err.downcast_ref::() { - if e == &StorageError::DoneForNow { - return Ok(()); - } - } - - return Err(err); - } - } - - Ok(()) - } - - pub fn is_latest_delete_marker(buf: &[u8]) -> bool { - let header = Self::decode_xl_headers(buf).ok(); - if let Some((versions, _hdr_v, _meta_v, meta)) = header { - if versions == 0 { - return false; - } - - let mut is_delete_marker = false; - - let _ = Self::decode_versions(meta, versions, |_: usize, hdr: &[u8], _: &[u8]| { - let mut header = FileMetaVersionHeader::default(); - if header.unmarshal_msg(hdr).is_err() { - return Err(Error::new(StorageError::DoneForNow)); - } - - is_delete_marker = header.version_type == VersionType::Delete; - - Err(Error::new(StorageError::DoneForNow)) - }); - - is_delete_marker - } else { - false - } - } - - #[tracing::instrument(level = "debug", skip_all)] - pub fn marshal_msg(&self) -> Result> { - let mut wr = Vec::new(); - - // header - wr.write_all(XL_FILE_HEADER.as_slice())?; - - let mut major = [0u8; 2]; - byteorder::LittleEndian::write_u16(&mut major, XL_FILE_VERSION_MAJOR); - wr.write_all(major.as_slice())?; - - let mut minor = [0u8; 2]; - byteorder::LittleEndian::write_u16(&mut minor, XL_FILE_VERSION_MINOR); - wr.write_all(minor.as_slice())?; - - // size bin32 预留 write_bin_len - wr.write_all(&[0xc6, 0, 0, 0, 0])?; - - let offset = wr.len(); - - rmp::encode::write_uint8(&mut wr, XL_HEADER_VERSION)?; - rmp::encode::write_uint8(&mut wr, XL_META_VERSION)?; - - // versions - rmp::encode::write_sint(&mut wr, self.versions.len() as i64)?; - - for ver in self.versions.iter() { - let hmsg = ver.header.marshal_msg()?; - rmp::encode::write_bin(&mut wr, &hmsg)?; - - rmp::encode::write_bin(&mut wr, &ver.meta)?; - } - - // 更新 bin 长度 - let data_len = wr.len() - offset; - byteorder::BigEndian::write_u32(&mut wr[offset - 4..offset], data_len as u32); - - let crc = xxh64::xxh64(&wr[offset..], XXHASH_SEED) as u32; - let mut crc_buf = [0u8; 5]; - crc_buf[0] = 0xce; // u32 - byteorder::BigEndian::write_u32(&mut crc_buf[1..], crc); - - wr.write_all(&crc_buf)?; - - wr.write_all(self.data.as_slice())?; - - Ok(wr) - } - - // pub fn unmarshal(buf: &[u8]) -> Result { - // let mut s = Self::default(); - // s.unmarshal_msg(buf)?; - // Ok(s) - // // let t: FileMeta = rmp_serde::from_slice(buf)?; - // // Ok(t) - // } - - // pub fn marshal_msg(&self) -> Result> { - // let mut buf = Vec::new(); - - // self.serialize(&mut Serializer::new(&mut buf))?; - - // Ok(buf) - // } - - fn get_idx(&self, idx: usize) -> Result { - if idx > self.versions.len() { - return Err(Error::new(DiskError::FileNotFound)); - } - - FileMetaVersion::try_from(self.versions[idx].meta.as_slice()) - } - - fn set_idx(&mut self, idx: usize, ver: FileMetaVersion) -> Result<()> { - if idx >= self.versions.len() { - return Err(Error::new(DiskError::FileNotFound)); - } - - // TODO: use old buf - let meta_buf = ver.marshal_msg()?; - - let pre_mod_time = self.versions[idx].header.mod_time; - - self.versions[idx].header = ver.header(); - self.versions[idx].meta = meta_buf; - - if pre_mod_time != self.versions[idx].header.mod_time { - self.sort_by_mod_time(); - } - - Ok(()) - } - - fn sort_by_mod_time(&mut self) { - if self.versions.len() <= 1 { - return; - } - - // Sort by mod_time in descending order (latest first) - self.versions.sort_by(|a, b| { - match (a.header.mod_time, b.header.mod_time) { - (Some(a_time), Some(b_time)) => b_time.cmp(&a_time), // Descending order - (Some(_), None) => Ordering::Less, - (None, Some(_)) => Ordering::Greater, - (None, None) => Ordering::Equal, - } - }); - } - - // 查找版本 - pub fn find_version(&self, vid: Option) -> Result<(usize, FileMetaVersion)> { - for (i, fver) in self.versions.iter().enumerate() { - if fver.header.version_id == vid { - let version = self.get_idx(i)?; - return Ok((i, version)); - } - } - - Err(Error::new(DiskError::FileVersionNotFound)) - } - - // shard_data_dir_count 查询 vid 下 data_dir 的数量 - #[tracing::instrument(level = "debug", skip_all)] - pub fn shard_data_dir_count(&self, vid: &Option, data_dir: &Option) -> usize { - self.versions - .iter() - .filter(|v| v.header.version_type == VersionType::Object && v.header.version_id != *vid && v.header.user_data_dir()) - .map(|v| FileMetaVersion::decode_data_dir_from_meta(&v.meta).unwrap_or_default()) - .filter(|v| v == data_dir) - .count() - } - - pub fn update_object_version(&mut self, fi: FileInfo) -> Result<()> { - for version in self.versions.iter_mut() { - match version.header.version_type { - VersionType::Invalid => (), - VersionType::Object => { - if version.header.version_id == fi.version_id { - let mut ver = FileMetaVersion::try_from(version.meta.as_slice())?; - - if let Some(ref mut obj) = ver.object { - if let Some(ref mut meta_user) = obj.meta_user { - if let Some(meta) = &fi.metadata { - for (k, v) in meta { - meta_user.insert(k.clone(), v.clone()); - } - } - obj.meta_user = Some(meta_user.clone()); - } else { - let mut meta_user = HashMap::new(); - if let Some(meta) = &fi.metadata { - for (k, v) in meta { - // TODO: MetaSys - meta_user.insert(k.clone(), v.clone()); - } - } - obj.meta_user = Some(meta_user); - } - - if let Some(mod_time) = fi.mod_time { - obj.mod_time = Some(mod_time); - } - } - - // 更新 - version.header = ver.header(); - version.meta = ver.marshal_msg()?; - } - } - VersionType::Delete => { - if version.header.version_id == fi.version_id { - return Err(Error::msg("method not allowed")); - } - } - } - } - - self.versions.sort_by(|a, b| { - if a.header.mod_time != b.header.mod_time { - a.header.mod_time.cmp(&b.header.mod_time) - } else if a.header.version_type != b.header.version_type { - a.header.version_type.cmp(&b.header.version_type) - } else if a.header.version_id != b.header.version_id { - a.header.version_id.cmp(&b.header.version_id) - } else if a.header.flags != b.header.flags { - a.header.flags.cmp(&b.header.flags) - } else { - a.cmp(b) - } - }); - Ok(()) - } - - // 添加版本 - #[tracing::instrument(level = "debug", skip_all)] - pub fn add_version(&mut self, fi: FileInfo) -> Result<()> { - let vid = fi.version_id; - - if let Some(ref data) = fi.data { - let key = vid.unwrap_or_default().to_string(); - self.data.replace(&key, data.clone())?; - } - - let version = FileMetaVersion::from(fi); - - if !version.valid() { - return Err(Error::msg("file meta version invalid")); - } - - // should replace - for (idx, ver) in self.versions.iter().enumerate() { - if ver.header.version_id != vid { - continue; - } - - return self.set_idx(idx, version); - } - - // TODO: version count limit ! - - let mod_time = version.get_mod_time(); - - // puth a -1 mod time value , so we can relplace this - self.versions.push(FileMetaShallowVersion { - header: FileMetaVersionHeader { - mod_time: Some(OffsetDateTime::from_unix_timestamp(-1)?), - ..Default::default() - }, - ..Default::default() - }); - - for (idx, exist) in self.versions.iter().enumerate() { - if let Some(ref ex_mt) = exist.header.mod_time { - if let Some(ref in_md) = mod_time { - if ex_mt <= in_md { - // insert - self.versions.insert(idx, FileMetaShallowVersion::try_from(version)?); - self.versions.pop(); - return Ok(()); - } - } - } - } - - Err(Error::msg("add_version failed")) - } - - // delete_version 删除版本，返回 data_dir - pub fn delete_version(&mut self, fi: &FileInfo) -> Result> { - let mut ventry = FileMetaVersion::default(); - if fi.deleted { - ventry.version_type = VersionType::Delete; - ventry.delete_marker = Some(MetaDeleteMarker { - version_id: fi.version_id, - mod_time: fi.mod_time, - ..Default::default() - }); - - if !fi.is_valid() { - return Err(Error::msg("invalid file meta version")); - } - } - - for (i, ver) in self.versions.iter().enumerate() { - if ver.header.version_id != fi.version_id { - continue; - } - - return match ver.header.version_type { - VersionType::Invalid => Err(Error::msg("invalid file meta version")), - VersionType::Delete => Ok(None), - VersionType::Object => { - let v = self.get_idx(i)?; - - self.versions.remove(i); - - let a = v.object.map(|v| v.data_dir).unwrap_or_default(); - Ok(a) - } - }; - } - - Err(Error::new(DiskError::FileVersionNotFound)) - } - - // read_data fill fi.dada - #[tracing::instrument(level = "debug", skip(self))] - pub fn into_fileinfo( - &self, - volume: &str, - path: &str, - version_id: &str, - read_data: bool, - all_parts: bool, - ) -> Result { - let has_vid = { - if !version_id.is_empty() { - let id = Uuid::parse_str(version_id)?; - if !id.is_nil() { Some(id) } else { None } - } else { - None - } - }; - - let mut is_latest = true; - let mut succ_mod_time = None; - for ver in self.versions.iter() { - let header = &ver.header; - - if let Some(vid) = has_vid { - if header.version_id != Some(vid) { - is_latest = false; - succ_mod_time = header.mod_time; - continue; - } - } - - let mut fi = ver.to_fileinfo(volume, path, has_vid, all_parts)?; - fi.is_latest = is_latest; - if let Some(_d) = succ_mod_time { - fi.successor_mod_time = succ_mod_time; - } - if read_data { - fi.data = self.data.find(fi.version_id.unwrap_or_default().to_string().as_str())?; - } - - fi.num_versions = self.versions.len(); - - return Ok(fi); - } - - if has_vid.is_none() { - Err(Error::from(DiskError::FileNotFound)) - } else { - Err(Error::from(DiskError::FileVersionNotFound)) - } - } - - #[tracing::instrument(level = "debug", skip(self))] - pub fn into_file_info_versions(&self, volume: &str, path: &str, all_parts: bool) -> Result { - let mut versions = Vec::new(); - for version in self.versions.iter() { - let mut file_version = FileMetaVersion::default(); - file_version.unmarshal_msg(&version.meta)?; - let fi = file_version.to_fileinfo(volume, path, None, all_parts); - versions.push(fi); - } - - Ok(FileInfoVersions { - volume: volume.to_string(), - name: path.to_string(), - latest_mod_time: versions[0].mod_time, - versions, - ..Default::default() - }) - } - - pub fn lastest_mod_time(&self) -> Option { - if self.versions.is_empty() { - return None; - } - - self.versions.first().unwrap().header.mod_time - } -} - -// impl Display for FileMeta { -// fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { -// f.write_str("FileMeta:")?; -// for (i, ver) in self.versions.iter().enumerate() { -// let mut meta = FileMetaVersion::default(); -// meta.unmarshal_msg(&ver.meta).unwrap_or_default(); -// f.write_fmt(format_args!("ver:{} header {:?}, meta {:?}", i, ver.header, meta))?; -// } - -// f.write_str("\n") -// } -// } - -#[derive(Serialize, Deserialize, Debug, Default, PartialEq, Clone, Eq, PartialOrd, Ord)] -pub struct FileMetaShallowVersion { - pub header: FileMetaVersionHeader, - pub meta: Vec, // FileMetaVersion.marshal_msg -} - -impl FileMetaShallowVersion { - pub fn to_fileinfo(&self, volume: &str, path: &str, version_id: Option, all_parts: bool) -> Result { - let file_version = FileMetaVersion::try_from(self.meta.as_slice())?; - - Ok(file_version.to_fileinfo(volume, path, version_id, all_parts)) - } -} - -impl TryFrom for FileMetaShallowVersion { - type Error = Error; - - fn try_from(value: FileMetaVersion) -> std::result::Result { - let header = value.header(); - let meta = value.marshal_msg()?; - Ok(Self { meta, header }) - } -} - -#[derive(Serialize, Deserialize, Debug, Default, Clone, PartialEq)] -pub struct FileMetaVersion { - pub version_type: VersionType, - pub object: Option, - pub delete_marker: Option, - pub write_version: u64, // rustfs version -} - -impl FileMetaVersion { - pub fn valid(&self) -> bool { - if !self.version_type.valid() { - return false; - } - - match self.version_type { - VersionType::Object => self - .object - .as_ref() - .map(|v| v.erasure_algorithm.valid() && v.bitrot_checksum_algo.valid() && v.mod_time.is_some()) - .unwrap_or_default(), - VersionType::Delete => self - .delete_marker - .as_ref() - .map(|v| v.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH) > OffsetDateTime::UNIX_EPOCH) - .unwrap_or_default(), - _ => false, - } - } - - pub fn get_data_dir(&self) -> Option { - self.valid() - .then(|| { - if self.version_type == VersionType::Object { - self.object.as_ref().map(|v| v.data_dir).unwrap_or_default() - } else { - None - } - }) - .unwrap_or_default() - } - - pub fn get_version_id(&self) -> Option { - match self.version_type { - VersionType::Object | VersionType::Delete => self.object.as_ref().map(|v| v.version_id).unwrap_or_default(), - _ => None, - } - } - - pub fn get_mod_time(&self) -> Option { - match self.version_type { - VersionType::Object => self.object.as_ref().map(|v| v.mod_time).unwrap_or_default(), - VersionType::Delete => self.delete_marker.as_ref().map(|v| v.mod_time).unwrap_or_default(), - _ => None, - } - } - - // decode_data_dir_from_meta 从 meta 中读取 data_dir TODO: 直接从 meta buf 中只解析出 data_dir, msg.skip - pub fn decode_data_dir_from_meta(buf: &[u8]) -> Result> { - let mut ver = Self::default(); - ver.unmarshal_msg(buf)?; - - let data_dir = ver.object.map(|v| v.data_dir).unwrap_or_default(); - Ok(data_dir) - } - - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - while fields_len > 0 { - fields_len -= 1; - - // println!("unmarshal_msg fields idx {}", fields_len); - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - // println!("unmarshal_msg fields name len() {}", &str_len); - - // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let field = String::from_utf8(field_buff)?; - - // println!("unmarshal_msg fields name {}", &field); - - match field.as_str() { - "Type" => { - let u: u8 = rmp::decode::read_int(&mut cur)?; - self.version_type = VersionType::from_u8(u); - } - - "V2Obj" => { - // is_nil() - if buf[cur.position() as usize] == 0xc0 { - rmp::decode::read_nil(&mut cur)?; - } else { - // let buf = unsafe { cur.position() }; - let mut obj = MetaObject::default(); - // let start = cur.position(); - - let (_, remain) = buf.split_at(cur.position() as usize); - - let read_len = obj.unmarshal_msg(remain)?; - cur.set_position(cur.position() + read_len); - - self.object = Some(obj); - } - } - "DelObj" => { - if buf[cur.position() as usize] == 0xc0 { - rmp::decode::read_nil(&mut cur)?; - } else { - // let buf = unsafe { cur.position() }; - let mut obj = MetaDeleteMarker::default(); - // let start = cur.position(); - - let (_, remain) = buf.split_at(cur.position() as usize); - let read_len = obj.unmarshal_msg(remain)?; - cur.set_position(cur.position() + read_len); - - self.delete_marker = Some(obj); - } - } - "v" => { - self.write_version = rmp::decode::read_int(&mut cur)?; - } - name => return Err(Error::msg(format!("not suport field name {}", name))), - } - } - - Ok(cur.position()) - } - - pub fn marshal_msg(&self) -> Result> { - let mut len: u32 = 4; - let mut mask: u8 = 0; - - if self.object.is_none() { - len -= 1; - mask |= 0x2; - } - if self.delete_marker.is_none() { - len -= 1; - mask |= 0x4; - } - - let mut wr = Vec::new(); - - // 字段数量 - rmp::encode::write_map_len(&mut wr, len)?; - - // write "Type" - rmp::encode::write_str(&mut wr, "Type")?; - rmp::encode::write_uint(&mut wr, self.version_type.to_u8() as u64)?; - - if (mask & 0x2) == 0 { - // write V2Obj - rmp::encode::write_str(&mut wr, "V2Obj")?; - if self.object.is_none() { - let _ = rmp::encode::write_nil(&mut wr); - } else { - let buf = self.object.as_ref().unwrap().marshal_msg()?; - wr.write_all(&buf)?; - } - } - - if (mask & 0x4) == 0 { - // write "DelObj" - rmp::encode::write_str(&mut wr, "DelObj")?; - if self.delete_marker.is_none() { - let _ = rmp::encode::write_nil(&mut wr); - } else { - let buf = self.delete_marker.as_ref().unwrap().marshal_msg()?; - wr.write_all(&buf)?; - } - } - - // write "v" - rmp::encode::write_str(&mut wr, "v")?; - rmp::encode::write_uint(&mut wr, self.write_version)?; - - Ok(wr) - } - - pub fn free_version(&self) -> bool { - self.version_type == VersionType::Delete && self.delete_marker.as_ref().map(|m| m.free_version()).unwrap_or_default() - } - - pub fn header(&self) -> FileMetaVersionHeader { - FileMetaVersionHeader::from(self.clone()) - } - - pub fn to_fileinfo(&self, volume: &str, path: &str, version_id: Option, all_parts: bool) -> FileInfo { - match self.version_type { - VersionType::Invalid => FileInfo { - name: path.to_string(), - volume: volume.to_string(), - version_id, - ..Default::default() - }, - VersionType::Object => self - .object - .as_ref() - .unwrap() - .clone() - .into_fileinfo(volume, path, version_id, all_parts), - VersionType::Delete => self - .delete_marker - .as_ref() - .unwrap() - .clone() - .into_fileinfo(volume, path, version_id, all_parts), - } - } -} - -impl TryFrom<&[u8]> for FileMetaVersion { - type Error = Error; - - fn try_from(value: &[u8]) -> std::result::Result { - let mut ver = FileMetaVersion::default(); - ver.unmarshal_msg(value)?; - Ok(ver) - } -} - -impl From for FileMetaVersion { - fn from(value: FileInfo) -> Self { - { - if value.deleted { - FileMetaVersion { - version_type: VersionType::Delete, - delete_marker: Some(MetaDeleteMarker::from(value)), - object: None, - write_version: 0, - } - } else { - FileMetaVersion { - version_type: VersionType::Object, - delete_marker: None, - object: Some(MetaObject::from(value)), - write_version: 0, - } - } - } - } -} - -impl TryFrom for FileMetaVersion { - type Error = Error; - - fn try_from(value: FileMetaShallowVersion) -> std::result::Result { - FileMetaVersion::try_from(value.meta.as_slice()) - } -} - -#[derive(Serialize, Deserialize, Debug, PartialEq, Default, Clone, Eq, Hash)] -pub struct FileMetaVersionHeader { - pub version_id: Option, - pub mod_time: Option, - pub signature: [u8; 4], - pub version_type: VersionType, - pub flags: u8, - pub ec_n: u8, - pub ec_m: u8, -} - -impl FileMetaVersionHeader { - pub fn has_ec(&self) -> bool { - self.ec_m > 0 && self.ec_n > 0 - } - - pub fn matches_not_strict(&self, o: &FileMetaVersionHeader) -> bool { - let mut ok = self.version_id == o.version_id && self.version_type == o.version_type && self.matches_ec(o); - if self.version_id.is_none() { - ok = ok && self.mod_time == o.mod_time; - } - - ok - } - - pub fn matches_ec(&self, o: &FileMetaVersionHeader) -> bool { - if self.has_ec() && o.has_ec() { - return self.ec_n == o.ec_n && self.ec_m == o.ec_m; - } - - true - } - - pub fn free_version(&self) -> bool { - self.flags & XL_FLAG_FREE_VERSION != 0 - } - - pub fn sorts_before(&self, o: &FileMetaVersionHeader) -> bool { - if self == o { - return false; - } - - // Prefer newest modtime. - if self.mod_time != o.mod_time { - return self.mod_time > o.mod_time; - } - - match self.mod_time.cmp(&o.mod_time) { - Ordering::Greater => { - return true; - } - Ordering::Less => { - return false; - } - _ => {} - } - - // The following doesn't make too much sense, but we want sort to be consistent nonetheless. - // Prefer lower types - if self.version_type != o.version_type { - return self.version_type < o.version_type; - } - // Consistent sort on signature - match self.version_id.cmp(&o.version_id) { - Ordering::Greater => { - return true; - } - Ordering::Less => { - return false; - } - _ => {} - } - - if self.flags != o.flags { - return self.flags > o.flags; - } - - false - } - - pub fn user_data_dir(&self) -> bool { - self.flags & Flags::UsesDataDir as u8 != 0 - } - #[tracing::instrument] - pub fn marshal_msg(&self) -> Result> { - let mut wr = Vec::new(); - - // array len 7 - rmp::encode::write_array_len(&mut wr, 7)?; - - // version_id - rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?; - // mod_time - rmp::encode::write_i64(&mut wr, self.mod_time.unwrap_or(OffsetDateTime::UNIX_EPOCH).unix_timestamp_nanos() as i64)?; - // signature - rmp::encode::write_bin(&mut wr, self.signature.as_slice())?; - // version_type - rmp::encode::write_uint8(&mut wr, self.version_type.to_u8())?; - // flags - rmp::encode::write_uint8(&mut wr, self.flags)?; - // ec_n - rmp::encode::write_uint8(&mut wr, self.ec_n)?; - // ec_m - rmp::encode::write_uint8(&mut wr, self.ec_m)?; - - Ok(wr) - } - - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let mut cur = Cursor::new(buf); - let alen = rmp::decode::read_array_len(&mut cur)?; - if alen != 7 { - return Err(Error::msg(format!("version header array len err need 7 got {}", alen))); - } - - // version_id - rmp::decode::read_bin_len(&mut cur)?; - let mut buf = [0u8; 16]; - cur.read_exact(&mut buf)?; - self.version_id = { - let id = Uuid::from_bytes(buf); - if id.is_nil() { None } else { Some(id) } - }; - - // mod_time - let unix: i128 = rmp::decode::read_int(&mut cur)?; - - let time = OffsetDateTime::from_unix_timestamp_nanos(unix)?; - if time == OffsetDateTime::UNIX_EPOCH { - self.mod_time = None; - } else { - self.mod_time = Some(time); - } - - // signature - rmp::decode::read_bin_len(&mut cur)?; - cur.read_exact(&mut self.signature)?; - - // version_type - let typ: u8 = rmp::decode::read_int(&mut cur)?; - self.version_type = VersionType::from_u8(typ); - - // flags - self.flags = rmp::decode::read_int(&mut cur)?; - // ec_n - self.ec_n = rmp::decode::read_int(&mut cur)?; - // ec_m - self.ec_m = rmp::decode::read_int(&mut cur)?; - - Ok(cur.position()) - } -} - -impl PartialOrd for FileMetaVersionHeader { - fn partial_cmp(&self, other: &Self) -> Option { - Some(self.cmp(other)) - } -} - -impl Ord for FileMetaVersionHeader { - fn cmp(&self, other: &Self) -> Ordering { - match self.mod_time.cmp(&other.mod_time) { - Ordering::Equal => {} - ord => return ord, - } - - match self.version_type.cmp(&other.version_type) { - Ordering::Equal => {} - ord => return ord, - } - match self.signature.cmp(&other.signature) { - Ordering::Equal => {} - ord => return ord, - } - match self.version_id.cmp(&other.version_id) { - Ordering::Equal => {} - ord => return ord, - } - self.flags.cmp(&other.flags) - } -} - -impl From for FileMetaVersionHeader { - fn from(value: FileMetaVersion) -> Self { - let flags = { - let mut f: u8 = 0; - if value.free_version() { - f |= Flags::FreeVersion as u8; - } - - if value.version_type == VersionType::Object && value.object.as_ref().map(|v| v.use_data_dir()).unwrap_or_default() { - f |= Flags::UsesDataDir as u8; - } - - if value.version_type == VersionType::Object && value.object.as_ref().map(|v| v.use_inlinedata()).unwrap_or_default() - { - f |= Flags::InlineData as u8; - } - - f - }; - - let (ec_n, ec_m) = { - if value.version_type == VersionType::Object && value.object.is_some() { - ( - value.object.as_ref().unwrap().erasure_n as u8, - value.object.as_ref().unwrap().erasure_m as u8, - ) - } else { - (0, 0) - } - }; - - Self { - version_id: value.get_version_id(), - mod_time: value.get_mod_time(), - signature: [0, 0, 0, 0], - version_type: value.version_type, - flags, - ec_n, - ec_m, - } - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] -// 因为自定义 message_pack，所以一定要保证字段顺序 -pub struct MetaObject { - pub version_id: Option, // Version ID - pub data_dir: Option, // Data dir ID - pub erasure_algorithm: ErasureAlgo, // Erasure coding algorithm - pub erasure_m: usize, // Erasure data blocks - pub erasure_n: usize, // Erasure parity blocks - pub erasure_block_size: usize, // Erasure block size - pub erasure_index: usize, // Erasure disk index - pub erasure_dist: Vec, // Erasure distribution - pub bitrot_checksum_algo: ChecksumAlgo, // Bitrot checksum algo - pub part_numbers: Vec, // Part Numbers - pub part_etags: Option>, // Part ETags - pub part_sizes: Vec, // Part Sizes - pub part_actual_sizes: Option>, // Part ActualSizes (compression) - pub part_indices: Option>>, // Part Indexes (compression) - pub size: usize, // Object version size - pub mod_time: Option, // Object version modified time - pub meta_sys: Option>>, // Object version internal metadata - pub meta_user: Option>, // Object version metadata set by user -} - -impl MetaObject { - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - // let mut ret = Self::default(); - - while fields_len > 0 { - fields_len -= 1; - - // println!("unmarshal_msg fields idx {}", fields_len); - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - // println!("unmarshal_msg fields name len() {}", &str_len); - - // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let field = String::from_utf8(field_buff)?; - - // println!("unmarshal_msg fields name {}", &field); - - match field.as_str() { - "ID" => { - rmp::decode::read_bin_len(&mut cur)?; - let mut buf = [0u8; 16]; - cur.read_exact(&mut buf)?; - self.version_id = { - let id = Uuid::from_bytes(buf); - if id.is_nil() { None } else { Some(id) } - }; - } - "DDir" => { - rmp::decode::read_bin_len(&mut cur)?; - let mut buf = [0u8; 16]; - cur.read_exact(&mut buf)?; - self.data_dir = { - let id = Uuid::from_bytes(buf); - if id.is_nil() { None } else { Some(id) } - }; - } - "EcAlgo" => { - let u: u8 = rmp::decode::read_int(&mut cur)?; - self.erasure_algorithm = ErasureAlgo::from_u8(u) - } - "EcM" => { - self.erasure_m = rmp::decode::read_int(&mut cur)?; - } - "EcN" => { - self.erasure_n = rmp::decode::read_int(&mut cur)?; - } - "EcBSize" => { - self.erasure_block_size = rmp::decode::read_int(&mut cur)?; - } - "EcIndex" => { - self.erasure_index = rmp::decode::read_int(&mut cur)?; - } - "EcDist" => { - let alen = rmp::decode::read_array_len(&mut cur)? as usize; - self.erasure_dist = vec![0u8; alen]; - for i in 0..alen { - self.erasure_dist[i] = rmp::decode::read_int(&mut cur)?; - } - } - "CSumAlgo" => { - let u: u8 = rmp::decode::read_int(&mut cur)?; - self.bitrot_checksum_algo = ChecksumAlgo::from_u8(u) - } - "PartNums" => { - let alen = rmp::decode::read_array_len(&mut cur)? as usize; - self.part_numbers = vec![0; alen]; - for i in 0..alen { - self.part_numbers[i] = rmp::decode::read_int(&mut cur)?; - } - } - "PartETags" => { - let array_len = match rmp::decode::read_nil(&mut cur) { - Ok(_) => None, - Err(e) => match e { - rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { - Marker::FixArray(l) => Some(l as usize), - Marker::Array16 => Some(rmp::decode::read_u16(&mut cur)? as usize), - Marker::Array32 => Some(rmp::decode::read_u16(&mut cur)? as usize), - _ => return Err(Error::msg("PartETags parse failed")), - }, - _ => return Err(Error::msg("PartETags parse failed.")), - }, - }; - - if array_len.is_some() { - let l = array_len.unwrap(); - let mut etags = Vec::with_capacity(l); - for _ in 0..l { - let str_len = rmp::decode::read_str_len(&mut cur)?; - let mut field_buff = vec![0u8; str_len as usize]; - cur.read_exact(&mut field_buff)?; - etags.push(String::from_utf8(field_buff)?); - } - self.part_etags = Some(etags); - } - } - "PartSizes" => { - let alen = rmp::decode::read_array_len(&mut cur)? as usize; - self.part_sizes = vec![0; alen]; - for i in 0..alen { - self.part_sizes[i] = rmp::decode::read_int(&mut cur)?; - } - } - "PartASizes" => { - let array_len = match rmp::decode::read_nil(&mut cur) { - Ok(_) => None, - Err(e) => match e { - rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { - Marker::FixArray(l) => Some(l as usize), - Marker::Array16 => Some(rmp::decode::read_u16(&mut cur)? as usize), - Marker::Array32 => Some(rmp::decode::read_u16(&mut cur)? as usize), - _ => return Err(Error::msg("PartETags parse failed")), - }, - _ => return Err(Error::msg("PartETags parse failed.")), - }, - }; - if let Some(l) = array_len { - let mut sizes = vec![0; l]; - for size in sizes.iter_mut().take(l) { - *size = rmp::decode::read_int(&mut cur)?; - } - // for size in sizes.iter_mut().take(l) { - // let tmp = rmp::decode::read_int(&mut cur)?; - // size = tmp; - // } - self.part_actual_sizes = Some(sizes); - } - } - "PartIdx" => { - let alen = rmp::decode::read_array_len(&mut cur)? as usize; - - if alen == 0 { - self.part_indices = None; - continue; - } - - let mut indices = Vec::with_capacity(alen); - for _ in 0..alen { - let blen = rmp::decode::read_bin_len(&mut cur)?; - let mut buf = vec![0u8; blen as usize]; - cur.read_exact(&mut buf)?; - - indices.push(buf); - } - - self.part_indices = Some(indices); - } - "Size" => { - self.size = rmp::decode::read_int(&mut cur)?; - } - "MTime" => { - let unix: i128 = rmp::decode::read_int(&mut cur)?; - let time = OffsetDateTime::from_unix_timestamp_nanos(unix)?; - if time == OffsetDateTime::UNIX_EPOCH { - self.mod_time = None; - } else { - self.mod_time = Some(time); - } - } - "MetaSys" => { - let len = match rmp::decode::read_nil(&mut cur) { - Ok(_) => None, - Err(e) => match e { - rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { - Marker::FixMap(l) => Some(l as usize), - Marker::Map16 => Some(rmp::decode::read_u16(&mut cur)? as usize), - Marker::Map32 => Some(rmp::decode::read_u16(&mut cur)? as usize), - _ => return Err(Error::msg("MetaSys parse failed")), - }, - _ => return Err(Error::msg("MetaSys parse failed.")), - }, - }; - if len.is_some() { - let l = len.unwrap(); - let mut map = HashMap::new(); - for _ in 0..l { - let str_len = rmp::decode::read_str_len(&mut cur)?; - let mut field_buff = vec![0u8; str_len as usize]; - cur.read_exact(&mut field_buff)?; - let key = String::from_utf8(field_buff)?; - - let blen = rmp::decode::read_bin_len(&mut cur)?; - let mut val = vec![0u8; blen as usize]; - cur.read_exact(&mut val)?; - - map.insert(key, val); - } - - self.meta_sys = Some(map); - } - } - "MetaUsr" => { - let len = match rmp::decode::read_nil(&mut cur) { - Ok(_) => None, - Err(e) => match e { - rmp::decode::ValueReadError::TypeMismatch(marker) => match marker { - Marker::FixMap(l) => Some(l as usize), - Marker::Map16 => Some(rmp::decode::read_u16(&mut cur)? as usize), - Marker::Map32 => Some(rmp::decode::read_u16(&mut cur)? as usize), - _ => return Err(Error::msg("MetaUsr parse failed")), - }, - _ => return Err(Error::msg("MetaUsr parse failed.")), - }, - }; - if len.is_some() { - let l = len.unwrap(); - let mut map = HashMap::new(); - for _ in 0..l { - let str_len = rmp::decode::read_str_len(&mut cur)?; - let mut field_buff = vec![0u8; str_len as usize]; - cur.read_exact(&mut field_buff)?; - let key = String::from_utf8(field_buff)?; - - let blen = rmp::decode::read_str_len(&mut cur)?; - let mut val_buf = vec![0u8; blen as usize]; - cur.read_exact(&mut val_buf)?; - let val = String::from_utf8(val_buf)?; - - map.insert(key, val); - } - - self.meta_user = Some(map); - } - } - - name => return Err(Error::msg(format!("not suport field name {}", name))), - } - } - - Ok(cur.position()) - } - // marshal_msg 自定义 messagepack 命名与 go 一致 - pub fn marshal_msg(&self) -> Result> { - let mut len: u32 = 18; - let mut mask: u32 = 0; - - if self.part_indices.is_none() { - len -= 1; - mask |= 0x2000; - } - - let mut wr = Vec::new(); - - // 字段数量 - rmp::encode::write_map_len(&mut wr, len)?; - - // string "ID" - rmp::encode::write_str(&mut wr, "ID")?; - rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?; - - // string "DDir" - rmp::encode::write_str(&mut wr, "DDir")?; - rmp::encode::write_bin(&mut wr, self.data_dir.unwrap_or_default().as_bytes())?; - - // string "EcAlgo" - rmp::encode::write_str(&mut wr, "EcAlgo")?; - rmp::encode::write_uint(&mut wr, self.erasure_algorithm.to_u8() as u64)?; - - // string "EcM" - rmp::encode::write_str(&mut wr, "EcM")?; - rmp::encode::write_uint(&mut wr, self.erasure_m.try_into().unwrap())?; - - // string "EcN" - rmp::encode::write_str(&mut wr, "EcN")?; - rmp::encode::write_uint(&mut wr, self.erasure_n.try_into().unwrap())?; - - // string "EcBSize" - rmp::encode::write_str(&mut wr, "EcBSize")?; - rmp::encode::write_uint(&mut wr, self.erasure_block_size.try_into().unwrap())?; - - // string "EcIndex" - rmp::encode::write_str(&mut wr, "EcIndex")?; - rmp::encode::write_uint(&mut wr, self.erasure_index.try_into().unwrap())?; - - // string "EcDist" - rmp::encode::write_str(&mut wr, "EcDist")?; - rmp::encode::write_array_len(&mut wr, self.erasure_dist.len() as u32)?; - for v in self.erasure_dist.iter() { - rmp::encode::write_uint(&mut wr, *v as _)?; - } - - // string "CSumAlgo" - rmp::encode::write_str(&mut wr, "CSumAlgo")?; - rmp::encode::write_uint(&mut wr, self.bitrot_checksum_algo.to_u8() as u64)?; - - // string "PartNums" - rmp::encode::write_str(&mut wr, "PartNums")?; - rmp::encode::write_array_len(&mut wr, self.part_numbers.len() as u32)?; - for v in self.part_numbers.iter() { - rmp::encode::write_uint(&mut wr, *v as _)?; - } - - // string "PartETags" - rmp::encode::write_str(&mut wr, "PartETags")?; - if self.part_etags.is_none() { - rmp::encode::write_nil(&mut wr)?; - } else { - let etags = self.part_etags.as_ref().unwrap(); - rmp::encode::write_array_len(&mut wr, etags.len() as u32)?; - for v in etags.iter() { - rmp::encode::write_str(&mut wr, v.as_str())?; - } - } - - // string "PartSizes" - rmp::encode::write_str(&mut wr, "PartSizes")?; - rmp::encode::write_array_len(&mut wr, self.part_sizes.len() as u32)?; - for v in self.part_sizes.iter() { - rmp::encode::write_uint(&mut wr, *v as _)?; - } - - // string "PartASizes" - rmp::encode::write_str(&mut wr, "PartASizes")?; - if self.part_actual_sizes.is_none() { - rmp::encode::write_nil(&mut wr)?; - } else { - let asizes = self.part_actual_sizes.as_ref().unwrap(); - rmp::encode::write_array_len(&mut wr, asizes.len() as u32)?; - for v in asizes.iter() { - rmp::encode::write_uint(&mut wr, *v as _)?; - } - } - - if (mask & 0x2000) == 0 { - // string "PartIdx" - rmp::encode::write_str(&mut wr, "PartIdx")?; - let indices = self.part_indices.as_ref().unwrap(); - rmp::encode::write_array_len(&mut wr, indices.len() as u32)?; - for v in indices.iter() { - rmp::encode::write_bin(&mut wr, v)?; - } - } - - // string "Size" - rmp::encode::write_str(&mut wr, "Size")?; - rmp::encode::write_uint(&mut wr, self.size.try_into().unwrap())?; - - // string "MTime" - rmp::encode::write_str(&mut wr, "MTime")?; - rmp::encode::write_uint( - &mut wr, - self.mod_time - .unwrap_or(OffsetDateTime::UNIX_EPOCH) - .unix_timestamp_nanos() - .try_into() - .unwrap(), - )?; - - // string "MetaSys" - rmp::encode::write_str(&mut wr, "MetaSys")?; - if self.meta_sys.is_none() { - rmp::encode::write_nil(&mut wr)?; - } else { - let metas = self.meta_sys.as_ref().unwrap(); - rmp::encode::write_map_len(&mut wr, metas.len() as u32)?; - for (k, v) in metas { - rmp::encode::write_str(&mut wr, k.as_str())?; - rmp::encode::write_bin(&mut wr, v)?; - } - } - - // string "MetaUsr" - rmp::encode::write_str(&mut wr, "MetaUsr")?; - if self.meta_user.is_none() { - rmp::encode::write_nil(&mut wr)?; - } else { - let metas = self.meta_user.as_ref().unwrap(); - rmp::encode::write_map_len(&mut wr, metas.len() as u32)?; - for (k, v) in metas { - rmp::encode::write_str(&mut wr, k.as_str())?; - rmp::encode::write_str(&mut wr, v.as_str())?; - } - } - - Ok(wr) - } - pub fn use_data_dir(&self) -> bool { - // TODO: when use inlinedata - true - } - - pub fn use_inlinedata(&self) -> bool { - // TODO: when use inlinedata - false - } - - pub fn into_fileinfo(self, volume: &str, path: &str, _version_id: Option, _all_parts: bool) -> FileInfo { - let version_id = self.version_id; - - let erasure = ErasureInfo { - algorithm: self.erasure_algorithm.to_string(), - data_blocks: self.erasure_m, - parity_blocks: self.erasure_n, - block_size: self.erasure_block_size, - index: self.erasure_index, - distribution: self.erasure_dist.iter().map(|&v| v as usize).collect(), - ..Default::default() - }; - - let mut parts = Vec::new(); - for (i, _) in self.part_numbers.iter().enumerate() { - parts.push(ObjectPartInfo { - number: self.part_numbers[i], - size: self.part_sizes[i], - ..Default::default() - }); - } - - let metadata = { - if let Some(metauser) = self.meta_user.as_ref() { - let mut m = HashMap::new(); - for (k, v) in metauser { - // TODO: skip xhttp x-amz-storage-class - m.insert(k.to_owned(), v.to_owned()); - } - Some(m) - } else { - None - } - }; - - FileInfo { - version_id, - erasure, - data_dir: self.data_dir, - mod_time: self.mod_time, - size: self.size, - name: path.to_string(), - volume: volume.to_string(), - parts, - metadata, - ..Default::default() - } - } -} - -impl From for MetaObject { - fn from(value: FileInfo) -> Self { - let part_numbers: Vec = value.parts.iter().map(|v| v.number).collect(); - let part_sizes: Vec = value.parts.iter().map(|v| v.size).collect(); - - Self { - version_id: value.version_id, - size: value.size, - mod_time: value.mod_time, - data_dir: value.data_dir, - erasure_algorithm: ErasureAlgo::ReedSolomon, - erasure_m: value.erasure.data_blocks, - erasure_n: value.erasure.parity_blocks, - erasure_block_size: value.erasure.block_size, - erasure_index: value.erasure.index, - erasure_dist: value.erasure.distribution.iter().map(|x| *x as u8).collect(), - bitrot_checksum_algo: ChecksumAlgo::HighwayHash, - part_numbers, - part_etags: None, // TODO: add part_etags - part_sizes, - part_actual_sizes: None, // TODO: add part_etags - part_indices: None, - meta_sys: None, - meta_user: value.metadata.clone(), - } - } -} - -#[derive(Serialize, Deserialize, Debug, Clone, Default, PartialEq)] -pub struct MetaDeleteMarker { - pub version_id: Option, // Version ID for delete marker - pub mod_time: Option, // Object delete marker modified time - pub meta_sys: Option>>, // Delete marker internal metadata -} - -impl MetaDeleteMarker { - pub fn free_version(&self) -> bool { - self.meta_sys - .as_ref() - .map(|v| v.get(FREE_VERSION_META_HEADER).is_some()) - .unwrap_or_default() - } - - pub fn into_fileinfo(self, volume: &str, path: &str, version_id: Option, _all_parts: bool) -> FileInfo { - FileInfo { - name: path.to_string(), - volume: volume.to_string(), - version_id, - deleted: true, - mod_time: self.mod_time, - ..Default::default() - } - } - - pub fn unmarshal_msg(&mut self, buf: &[u8]) -> Result { - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - while fields_len > 0 { - fields_len -= 1; - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - // ！！！Vec::with_capacity(str_len) 失败，vec! 正常 - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let field = String::from_utf8(field_buff)?; - - match field.as_str() { - "ID" => { - rmp::decode::read_bin_len(&mut cur)?; - let mut buf = [0u8; 16]; - cur.read_exact(&mut buf)?; - self.version_id = { - let id = Uuid::from_bytes(buf); - if id.is_nil() { None } else { Some(id) } - }; - } - - "MTime" => { - let unix: i64 = rmp::decode::read_int(&mut cur)?; - let time = OffsetDateTime::from_unix_timestamp(unix)?; - if time == OffsetDateTime::UNIX_EPOCH { - self.mod_time = None; - } else { - self.mod_time = Some(time); - } - } - "MetaSys" => { - let l = rmp::decode::read_map_len(&mut cur)?; - let mut map = HashMap::new(); - for _ in 0..l { - let str_len = rmp::decode::read_str_len(&mut cur)?; - let mut field_buff = vec![0u8; str_len as usize]; - cur.read_exact(&mut field_buff)?; - let key = String::from_utf8(field_buff)?; - - let blen = rmp::decode::read_bin_len(&mut cur)?; - let mut val = vec![0u8; blen as usize]; - cur.read_exact(&mut val)?; - - map.insert(key, val); - } - - self.meta_sys = Some(map); - } - name => return Err(Error::msg(format!("not suport field name {}", name))), - } - } - - Ok(cur.position()) - } - - pub fn marshal_msg(&self) -> Result> { - let mut len: u32 = 3; - let mut mask: u8 = 0; - - if self.meta_sys.is_none() { - len -= 1; - mask |= 0x4; - } - - let mut wr = Vec::new(); - - // 字段数量 - rmp::encode::write_map_len(&mut wr, len)?; - - // string "ID" - rmp::encode::write_str(&mut wr, "ID")?; - rmp::encode::write_bin(&mut wr, self.version_id.unwrap_or_default().as_bytes())?; - - // string "MTime" - rmp::encode::write_str(&mut wr, "MTime")?; - rmp::encode::write_uint( - &mut wr, - self.mod_time - .unwrap_or(OffsetDateTime::UNIX_EPOCH) - .unix_timestamp() - .try_into() - .unwrap(), - )?; - - if (mask & 0x4) == 0 { - let metas = self.meta_sys.as_ref().unwrap(); - rmp::encode::write_map_len(&mut wr, metas.len() as u32)?; - for (k, v) in metas { - rmp::encode::write_str(&mut wr, k.as_str())?; - rmp::encode::write_bin(&mut wr, v)?; - } - } - - Ok(wr) - } -} - -impl From for MetaDeleteMarker { - fn from(value: FileInfo) -> Self { - Self { - version_id: value.version_id, - mod_time: value.mod_time, - meta_sys: None, - } - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, Default, Clone, PartialOrd, Ord, Hash)] -pub enum VersionType { - #[default] - Invalid = 0, - Object = 1, - Delete = 2, - // Legacy = 3, -} - -impl VersionType { - pub fn valid(&self) -> bool { - matches!(*self, VersionType::Object | VersionType::Delete) - } - - pub fn to_u8(&self) -> u8 { - match self { - VersionType::Invalid => 0, - VersionType::Object => 1, - VersionType::Delete => 2, - } - } - - pub fn from_u8(n: u8) -> Self { - match n { - 1 => VersionType::Object, - 2 => VersionType::Delete, - _ => VersionType::Invalid, - } - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)] -pub enum ErasureAlgo { - #[default] - Invalid = 0, - ReedSolomon = 1, -} - -impl ErasureAlgo { - pub fn valid(&self) -> bool { - *self > ErasureAlgo::Invalid - } - pub fn to_u8(&self) -> u8 { - match self { - ErasureAlgo::Invalid => 0, - ErasureAlgo::ReedSolomon => 1, - } - } - - pub fn from_u8(u: u8) -> Self { - match u { - 1 => ErasureAlgo::ReedSolomon, - _ => ErasureAlgo::Invalid, - } - } -} - -impl Display for ErasureAlgo { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match self { - ErasureAlgo::Invalid => write!(f, "Invalid"), - ErasureAlgo::ReedSolomon => write!(f, "{}", ERASURE_ALGORITHM), - } - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)] -pub enum ChecksumAlgo { - #[default] - Invalid = 0, - HighwayHash = 1, -} - -impl ChecksumAlgo { - pub fn valid(&self) -> bool { - *self > ChecksumAlgo::Invalid - } - pub fn to_u8(&self) -> u8 { - match self { - ChecksumAlgo::Invalid => 0, - ChecksumAlgo::HighwayHash => 1, - } - } - pub fn from_u8(u: u8) -> Self { - match u { - 1 => ChecksumAlgo::HighwayHash, - _ => ChecksumAlgo::Invalid, - } - } -} - -#[derive(Debug, Serialize, Deserialize, PartialEq, Eq, PartialOrd, Default, Clone)] -pub enum Flags { - #[default] - FreeVersion = 1 << 0, - UsesDataDir = 1 << 1, - InlineData = 1 << 2, -} - -const FREE_VERSION_META_HEADER: &str = "free-version"; - -// mergeXLV2Versions -pub fn merge_file_meta_versions( - mut quorum: usize, - mut strict: bool, - requested_versions: usize, - versions: &[Vec], -) -> Vec { - if quorum == 0 { - quorum = 1; - } - - if versions.len() < quorum || versions.is_empty() { - return Vec::new(); - } - - if versions.len() == 1 { - return versions[0].clone(); - } - - if quorum == 1 { - strict = true; - } - - let mut versions = versions.to_owned(); - - let mut n_versions = 0; - - let mut merged = Vec::new(); - loop { - let mut tops = Vec::new(); - let mut top_sig = FileMetaVersionHeader::default(); - let mut consistent = true; - for vers in versions.iter() { - if vers.is_empty() { - consistent = false; - continue; - } - if tops.is_empty() { - consistent = true; - top_sig = vers[0].header.clone(); - } else { - consistent = consistent && vers[0].header == top_sig; - } - tops.push(vers[0].clone()); - } - - // check if done... - if tops.len() < quorum { - break; - } - - let mut latest = FileMetaShallowVersion::default(); - if consistent { - merged.push(tops[0].clone()); - if tops[0].header.free_version() { - n_versions += 1; - } - } else { - let mut lastest_count = 0; - for (i, ver) in tops.iter().enumerate() { - if ver.header == latest.header { - lastest_count += 1; - continue; - } - - if i == 0 || ver.header.sorts_before(&latest.header) { - if i == 0 || lastest_count == 0 { - lastest_count = 1; - } else if !strict && ver.header.matches_not_strict(&latest.header) { - lastest_count += 1; - } else { - lastest_count = 1; - } - latest = ver.clone(); - continue; - } - - // Mismatch, but older. - if lastest_count > 0 && !strict && ver.header.matches_not_strict(&latest.header) { - lastest_count += 1; - continue; - } - - if lastest_count > 0 && ver.header.version_id == latest.header.version_id { - let mut x: HashMap = HashMap::new(); - for a in tops.iter() { - if a.header.version_id != ver.header.version_id { - continue; - } - let mut a_clone = a.clone(); - if !strict { - a_clone.header.signature = [0; 4]; - } - *x.entry(a_clone.header).or_insert(1) += 1; - } - lastest_count = 0; - for (k, v) in x.iter() { - if *v < lastest_count { - continue; - } - if *v == lastest_count && latest.header.sorts_before(k) { - continue; - } - tops.iter().for_each(|a| { - let mut hdr = a.header.clone(); - if !strict { - hdr.signature = [0; 4]; - } - if hdr == *k { - latest = a.clone(); - } - }); - - lastest_count = *v; - } - break; - } - } - if lastest_count >= quorum { - if !latest.header.free_version() { - n_versions += 1; - } - merged.push(latest.clone()); - } - } - - // Remove from all streams up until latest modtime or if selected. - versions.iter_mut().for_each(|vers| { - // // Keep top entry (and remaining)... - let mut bre = false; - vers.retain(|ver| { - if bre { - return true; - } - if let Ordering::Greater = ver.header.mod_time.cmp(&latest.header.mod_time) { - bre = true; - return false; - } - if ver.header == latest.header { - bre = true; - return false; - } - if let Ordering::Equal = latest.header.version_id.cmp(&ver.header.version_id) { - bre = true; - return false; - } - for merged_v in merged.iter() { - if let Ordering::Equal = ver.header.version_id.cmp(&merged_v.header.version_id) { - bre = true; - return false; - } - } - true - }); - }); - if requested_versions > 0 && requested_versions == n_versions { - merged.append(&mut versions[0]); - break; - } - } - - // Sanity check. Enable if duplicates show up. - // todo - merged -} - -pub async fn file_info_from_raw(ri: RawFileInfo, bucket: &str, object: &str, read_data: bool) -> Result { - get_file_info(&ri.buf, bucket, object, "", FileInfoOpts { data: read_data }).await -} - -pub struct FileInfoOpts { - pub data: bool, -} - -pub async fn get_file_info(buf: &[u8], volume: &str, path: &str, version_id: &str, opts: FileInfoOpts) -> Result { - let vid = { - if version_id.is_empty() { - None - } else { - Some(Uuid::parse_str(version_id)?) - } - }; - - let meta = FileMeta::load(buf)?; - if meta.versions.is_empty() { - return Ok(FileInfo { - volume: volume.to_owned(), - name: path.to_owned(), - version_id: vid, - is_latest: true, - deleted: true, - mod_time: Some(OffsetDateTime::from_unix_timestamp(1)?), - ..Default::default() - }); - } - - let fi = meta.into_fileinfo(volume, path, version_id, opts.data, true)?; - Ok(fi) -} - -async fn read_more( - reader: &mut R, - buf: &mut Vec, - total_size: usize, - read_size: usize, - has_full: bool, -) -> Result<()> { - use tokio::io::AsyncReadExt; - let has = buf.len(); - - if has >= read_size { - return Ok(()); - } - - if has_full || read_size > total_size { - return Err(Error::new(io::Error::new(io::ErrorKind::UnexpectedEof, "Unexpected EOF"))); - } - - let extra = read_size - has; - if buf.capacity() >= read_size { - // Extend the buffer if we have enough space. - buf.resize(read_size, 0); - } else { - buf.extend(vec![0u8; extra]); - } - - reader.read_exact(&mut buf[has..]).await?; - Ok(()) -} - -pub async fn read_xl_meta_no_data(reader: &mut R, size: usize) -> Result> { - use tokio::io::AsyncReadExt; - - let mut initial = size; - let mut has_full = true; - - if initial > META_DATA_READ_DEFAULT { - initial = META_DATA_READ_DEFAULT; - has_full = false; - } - - let mut buf = vec![0u8; initial]; - reader.read_exact(&mut buf).await?; - - let (tmp_buf, major, minor) = FileMeta::check_xl2_v1(&buf)?; - - match major { - 1 => match minor { - 0 => { - read_more(reader, &mut buf, size, size, has_full).await?; - Ok(buf) - } - 1..=3 => { - let (sz, tmp_buf) = FileMeta::read_bytes_header(tmp_buf)?; - let mut want = sz as usize + (buf.len() - tmp_buf.len()); - - if minor < 2 { - read_more(reader, &mut buf, size, want, has_full).await?; - return Ok(buf[..want].to_vec()); - } - - let want_max = usize::min(want + MSGP_UINT32_SIZE, size); - read_more(reader, &mut buf, size, want_max, has_full).await?; - - if buf.len() < want { - error!("read_xl_meta_no_data buffer too small (length: {}, needed: {})", &buf.len(), want); - return Err(Error::new(DiskError::FileCorrupt)); - } - - let tmp = &buf[want..]; - let crc_size = 5; - let other_size = tmp.len() - crc_size; - - want += tmp.len() - other_size; - - Ok(buf[..want].to_vec()) - } - _ => Err(Error::new(io::Error::new(io::ErrorKind::InvalidData, "Unknown minor metadata version"))), - }, - _ => Err(Error::new(io::Error::new(io::ErrorKind::InvalidData, "Unknown major metadata version"))), - } -} -#[cfg(test)] -#[allow(clippy::field_reassign_with_default)] -mod test { - use super::*; - - #[test] - fn test_new_file_meta() { - let mut fm = FileMeta::new(); - - let (m, n) = (3, 2); - - for i in 0..5 { - let mut fi = FileInfo::new(i.to_string().as_str(), m, n); - fi.mod_time = Some(OffsetDateTime::now_utc()); - - fm.add_version(fi).unwrap(); - } - - let buff = fm.marshal_msg().unwrap(); - - let mut newfm = FileMeta::default(); - newfm.unmarshal_msg(&buff).unwrap(); - - assert_eq!(fm, newfm) - } - - #[test] - fn test_marshal_metaobject() { - let obj = MetaObject { - data_dir: Some(Uuid::new_v4()), - ..Default::default() - }; - - // println!("obj {:?}", &obj); - - let encoded = obj.marshal_msg().unwrap(); - - let mut obj2 = MetaObject::default(); - obj2.unmarshal_msg(&encoded).unwrap(); - - // println!("obj2 {:?}", &obj2); - - assert_eq!(obj, obj2); - assert_eq!(obj.data_dir, obj2.data_dir); - } - - #[test] - fn test_marshal_metadeletemarker() { - let obj = MetaDeleteMarker { - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - // println!("obj {:?}", &obj); - - let encoded = obj.marshal_msg().unwrap(); - - let mut obj2 = MetaDeleteMarker::default(); - obj2.unmarshal_msg(&encoded).unwrap(); - - // println!("obj2 {:?}", &obj2); - - assert_eq!(obj, obj2); - assert_eq!(obj.version_id, obj2.version_id); - } - - #[test] - #[tracing::instrument] - fn test_marshal_metaversion() { - let mut fi = FileInfo::new("test", 3, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(OffsetDateTime::now_utc().unix_timestamp()).unwrap()); - let mut obj = FileMetaVersion::from(fi); - obj.write_version = 110; - - // println!("obj {:?}", &obj); - - let encoded = obj.marshal_msg().unwrap(); - - let mut obj2 = FileMetaVersion::default(); - obj2.unmarshal_msg(&encoded).unwrap(); - - // println!("obj2 {:?}", &obj2); - - // 时间截不一致 - - - assert_eq!(obj, obj2); - assert_eq!(obj.get_version_id(), obj2.get_version_id()); - assert_eq!(obj.write_version, obj2.write_version); - assert_eq!(obj.write_version, 110); - } - - #[test] - #[tracing::instrument] - fn test_marshal_metaversionheader() { - let mut obj = FileMetaVersionHeader::default(); - let vid = Some(Uuid::new_v4()); - obj.version_id = vid; - - let encoded = obj.marshal_msg().unwrap(); - - let mut obj2 = FileMetaVersionHeader::default(); - obj2.unmarshal_msg(&encoded).unwrap(); - - // 时间截不一致 - - - assert_eq!(obj, obj2); - assert_eq!(obj.version_id, obj2.version_id); - assert_eq!(obj.version_id, vid); - } - - // New comprehensive tests for utility functions and validation - - #[test] - fn test_xl_file_header_constants() { - // Test XL file header constants - assert_eq!(XL_FILE_HEADER, [b'X', b'L', b'2', b' ']); - assert_eq!(XL_FILE_VERSION_MAJOR, 1); - assert_eq!(XL_FILE_VERSION_MINOR, 3); - assert_eq!(XL_HEADER_VERSION, 3); - assert_eq!(XL_META_VERSION, 2); - } - - #[test] - fn test_is_xl2_v1_format() { - // Test valid XL2 V1 format - let mut valid_buf = vec![0u8; 20]; - valid_buf[0..4].copy_from_slice(&XL_FILE_HEADER); - byteorder::LittleEndian::write_u16(&mut valid_buf[4..6], 1); - byteorder::LittleEndian::write_u16(&mut valid_buf[6..8], 0); - - assert!(FileMeta::is_xl2_v1_format(&valid_buf)); - - // Test invalid format - wrong header - let invalid_buf = vec![0u8; 20]; - assert!(!FileMeta::is_xl2_v1_format(&invalid_buf)); - - // Test buffer too small - let small_buf = vec![0u8; 4]; - assert!(!FileMeta::is_xl2_v1_format(&small_buf)); - } - - #[test] - fn test_check_xl2_v1() { - // Test valid XL2 V1 check - let mut valid_buf = vec![0u8; 20]; - valid_buf[0..4].copy_from_slice(&XL_FILE_HEADER); - byteorder::LittleEndian::write_u16(&mut valid_buf[4..6], 1); - byteorder::LittleEndian::write_u16(&mut valid_buf[6..8], 2); - - let result = FileMeta::check_xl2_v1(&valid_buf); - assert!(result.is_ok()); - let (remaining, major, minor) = result.unwrap(); - assert_eq!(major, 1); - assert_eq!(minor, 2); - assert_eq!(remaining.len(), 12); // 20 - 8 - - // Test buffer too small - let small_buf = vec![0u8; 4]; - assert!(FileMeta::check_xl2_v1(&small_buf).is_err()); - - // Test wrong header - let mut wrong_header = vec![0u8; 20]; - wrong_header[0..4].copy_from_slice(b"ABCD"); - assert!(FileMeta::check_xl2_v1(&wrong_header).is_err()); - - // Test version too high - let mut high_version = vec![0u8; 20]; - high_version[0..4].copy_from_slice(&XL_FILE_HEADER); - byteorder::LittleEndian::write_u16(&mut high_version[4..6], 99); - byteorder::LittleEndian::write_u16(&mut high_version[6..8], 0); - assert!(FileMeta::check_xl2_v1(&high_version).is_err()); - } - - #[test] - fn test_version_type_enum() { - // Test VersionType enum methods - assert!(VersionType::Object.valid()); - assert!(VersionType::Delete.valid()); - assert!(!VersionType::Invalid.valid()); - - assert_eq!(VersionType::Object.to_u8(), 1); - assert_eq!(VersionType::Delete.to_u8(), 2); - assert_eq!(VersionType::Invalid.to_u8(), 0); - - assert_eq!(VersionType::from_u8(1), VersionType::Object); - assert_eq!(VersionType::from_u8(2), VersionType::Delete); - assert_eq!(VersionType::from_u8(99), VersionType::Invalid); - } - - #[test] - fn test_erasure_algo_enum() { - // Test ErasureAlgo enum methods - assert!(ErasureAlgo::ReedSolomon.valid()); - assert!(!ErasureAlgo::Invalid.valid()); - - assert_eq!(ErasureAlgo::ReedSolomon.to_u8(), 1); - assert_eq!(ErasureAlgo::Invalid.to_u8(), 0); - - assert_eq!(ErasureAlgo::from_u8(1), ErasureAlgo::ReedSolomon); - assert_eq!(ErasureAlgo::from_u8(99), ErasureAlgo::Invalid); - - // Test Display trait - assert_eq!(format!("{}", ErasureAlgo::ReedSolomon), "rs-vandermonde"); - assert_eq!(format!("{}", ErasureAlgo::Invalid), "Invalid"); - } - - #[test] - fn test_checksum_algo_enum() { - // Test ChecksumAlgo enum methods - assert!(ChecksumAlgo::HighwayHash.valid()); - assert!(!ChecksumAlgo::Invalid.valid()); - - assert_eq!(ChecksumAlgo::HighwayHash.to_u8(), 1); - assert_eq!(ChecksumAlgo::Invalid.to_u8(), 0); - - assert_eq!(ChecksumAlgo::from_u8(1), ChecksumAlgo::HighwayHash); - assert_eq!(ChecksumAlgo::from_u8(99), ChecksumAlgo::Invalid); - } - - #[test] - fn test_file_meta_version_header_methods() { - let mut header = FileMetaVersionHeader { - ec_n: 4, - ec_m: 2, - flags: XL_FLAG_FREE_VERSION, - ..Default::default() - }; - - // Test has_ec - assert!(header.has_ec()); - - // Test free_version - assert!(header.free_version()); - - // Test user_data_dir (should be false by default) - assert!(!header.user_data_dir()); - - // Test with different flags - header.flags = 0; - assert!(!header.free_version()); - } - - #[test] - fn test_file_meta_version_header_comparison() { - let mut header1 = FileMetaVersionHeader { - mod_time: Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - let mut header2 = FileMetaVersionHeader { - mod_time: Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - // Test sorts_before - header2 should sort before header1 (newer mod_time) - assert!(!header1.sorts_before(&header2)); - assert!(header2.sorts_before(&header1)); - - // Test matches_not_strict - let header3 = header1.clone(); - assert!(header1.matches_not_strict(&header3)); - - // Test matches_ec - header1.ec_n = 4; - header1.ec_m = 2; - header2.ec_n = 4; - header2.ec_m = 2; - assert!(header1.matches_ec(&header2)); - - header2.ec_n = 6; - assert!(!header1.matches_ec(&header2)); - } - - #[test] - fn test_file_meta_version_methods() { - // Test with object version - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.data_dir = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - - let version = FileMetaVersion::from(fi.clone()); - - assert!(version.valid()); - assert_eq!(version.get_version_id(), fi.version_id); - assert_eq!(version.get_data_dir(), fi.data_dir); - assert_eq!(version.get_mod_time(), fi.mod_time); - assert!(!version.free_version()); - - // Test with delete marker - let mut delete_fi = FileInfo::new("test", 4, 2); - delete_fi.deleted = true; - delete_fi.version_id = Some(Uuid::new_v4()); - delete_fi.mod_time = Some(OffsetDateTime::now_utc()); - - let delete_version = FileMetaVersion::from(delete_fi); - assert!(delete_version.valid()); - assert_eq!(delete_version.version_type, VersionType::Delete); - } - - #[test] - fn test_meta_object_methods() { - let mut obj = MetaObject { - data_dir: Some(Uuid::new_v4()), - size: 1024, - ..Default::default() - }; - - // Test use_data_dir - assert!(obj.use_data_dir()); - - obj.data_dir = None; - assert!(obj.use_data_dir()); // use_data_dir always returns true - - // Test use_inlinedata (currently always returns false) - obj.size = 100; // Small size - assert!(!obj.use_inlinedata()); - - obj.size = 100000; // Large size - assert!(!obj.use_inlinedata()); - } - - #[test] - fn test_meta_delete_marker_methods() { - let marker = MetaDeleteMarker::default(); - - // Test free_version (should always return false for delete markers) - assert!(!marker.free_version()); - } - - #[test] - fn test_file_meta_latest_mod_time() { - let mut fm = FileMeta::new(); - - // Empty FileMeta should return None - assert!(fm.lastest_mod_time().is_none()); - - // Add versions with different mod times - let time1 = OffsetDateTime::from_unix_timestamp(1000).unwrap(); - let time2 = OffsetDateTime::from_unix_timestamp(2000).unwrap(); - let time3 = OffsetDateTime::from_unix_timestamp(1500).unwrap(); - - let mut fi1 = FileInfo::new("test1", 4, 2); - fi1.mod_time = Some(time1); - fm.add_version(fi1).unwrap(); - - let mut fi2 = FileInfo::new("test2", 4, 2); - fi2.mod_time = Some(time2); - fm.add_version(fi2).unwrap(); - - let mut fi3 = FileInfo::new("test3", 4, 2); - fi3.mod_time = Some(time3); - fm.add_version(fi3).unwrap(); - - // Sort first to ensure latest is at the front - fm.sort_by_mod_time(); - - // Should return the first version's mod time (lastest_mod_time returns first version's time) - assert_eq!(fm.lastest_mod_time(), fm.versions[0].header.mod_time); - } - - #[test] - fn test_file_meta_shard_data_dir_count() { - let mut fm = FileMeta::new(); - let data_dir = Some(Uuid::new_v4()); - - // Add versions with same data_dir - for i in 0..3 { - let mut fi = FileInfo::new(&format!("test{}", i), 4, 2); - fi.data_dir = data_dir; - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - } - - // Add one version with different data_dir - let mut fi_diff = FileInfo::new("test_diff", 4, 2); - fi_diff.data_dir = Some(Uuid::new_v4()); - fi_diff.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi_diff).unwrap(); - - // Count should be 0 because user_data_dir() requires UsesDataDir flag to be set - assert_eq!(fm.shard_data_dir_count(&None, &data_dir), 0); - - // Count should be 0 for non-existent data_dir - assert_eq!(fm.shard_data_dir_count(&None, &Some(Uuid::new_v4())), 0); - } - - #[test] - fn test_file_meta_sort_by_mod_time() { - let mut fm = FileMeta::new(); - - let time1 = OffsetDateTime::from_unix_timestamp(3000).unwrap(); - let time2 = OffsetDateTime::from_unix_timestamp(1000).unwrap(); - let time3 = OffsetDateTime::from_unix_timestamp(2000).unwrap(); - - // Add versions in non-chronological order - let mut fi1 = FileInfo::new("test1", 4, 2); - fi1.mod_time = Some(time1); - fm.add_version(fi1).unwrap(); - - let mut fi2 = FileInfo::new("test2", 4, 2); - fi2.mod_time = Some(time2); - fm.add_version(fi2).unwrap(); - - let mut fi3 = FileInfo::new("test3", 4, 2); - fi3.mod_time = Some(time3); - fm.add_version(fi3).unwrap(); - - // Sort by mod time - fm.sort_by_mod_time(); - - // Verify they are sorted (newest first) - add_version already sorts by insertion - // The actual order depends on how add_version inserts them - // Let's check the first version is the latest - let latest_time = fm.versions.iter().map(|v| v.header.mod_time).max().flatten(); - assert_eq!(fm.versions[0].header.mod_time, latest_time); - } - - #[test] - fn test_file_meta_find_version() { - let mut fm = FileMeta::new(); - let version_id = Some(Uuid::new_v4()); - - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = version_id; - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - // Should find the version - let result = fm.find_version(version_id); - assert!(result.is_ok()); - let (idx, version) = result.unwrap(); - assert_eq!(idx, 0); - assert_eq!(version.get_version_id(), version_id); - - // Should not find non-existent version - let non_existent_id = Some(Uuid::new_v4()); - assert!(fm.find_version(non_existent_id).is_err()); - } - - #[test] - fn test_file_meta_delete_version() { - let mut fm = FileMeta::new(); - let version_id = Some(Uuid::new_v4()); - - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = version_id; - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi.clone()).unwrap(); - - assert_eq!(fm.versions.len(), 1); - - // Delete the version - let result = fm.delete_version(&fi); - assert!(result.is_ok()); - - // Version should be removed - assert_eq!(fm.versions.len(), 0); - } - - #[test] - fn test_file_meta_update_object_version() { - let mut fm = FileMeta::new(); - let version_id = Some(Uuid::new_v4()); - - // Add initial version - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = version_id; - fi.size = 1024; - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi.clone()).unwrap(); - - // Update with new metadata (size is not updated by update_object_version) - let mut metadata = HashMap::new(); - metadata.insert("test-key".to_string(), "test-value".to_string()); - fi.metadata = Some(metadata.clone()); - let result = fm.update_object_version(fi); - assert!(result.is_ok()); - - // Verify the metadata was updated - let (_, updated_version) = fm.find_version(version_id).unwrap(); - if let Some(obj) = updated_version.object { - assert_eq!(obj.size, 1024); // Size remains unchanged - assert_eq!(obj.meta_user, Some(metadata)); // Metadata is updated - } else { - panic!("Expected object version"); - } - } - - #[test] - fn test_file_info_opts() { - let opts = FileInfoOpts { data: true }; - assert!(opts.data); - - let opts_no_data = FileInfoOpts { data: false }; - assert!(!opts_no_data.data); - } - - #[test] - fn test_decode_data_dir_from_meta() { - // Test with valid metadata containing data_dir - let data_dir = Some(Uuid::new_v4()); - let obj = MetaObject { - data_dir, - mod_time: Some(OffsetDateTime::now_utc()), - erasure_algorithm: ErasureAlgo::ReedSolomon, - bitrot_checksum_algo: ChecksumAlgo::HighwayHash, - ..Default::default() - }; - - // Create a valid FileMetaVersion with the object - let version = FileMetaVersion { - version_type: VersionType::Object, - object: Some(obj), - ..Default::default() - }; - - let encoded = version.marshal_msg().unwrap(); - let result = FileMetaVersion::decode_data_dir_from_meta(&encoded); - assert!(result.is_ok()); - assert_eq!(result.unwrap(), data_dir); - - // Test with invalid metadata - let invalid_data = vec![0u8; 10]; - let result = FileMetaVersion::decode_data_dir_from_meta(&invalid_data); - assert!(result.is_err()); - } - - #[test] - fn test_is_latest_delete_marker() { - // Test the is_latest_delete_marker function with simple data - // Since the function is complex and requires specific XL format, - // we'll test with empty data which should return false - let empty_data = vec![]; - assert!(!FileMeta::is_latest_delete_marker(&empty_data)); - - // Test with invalid data - let invalid_data = vec![1, 2, 3, 4, 5]; - assert!(!FileMeta::is_latest_delete_marker(&invalid_data)); - } - - #[test] - fn test_merge_file_meta_versions_basic() { - // Test basic merge functionality - let mut version1 = FileMetaShallowVersion::default(); - version1.header.version_id = Some(Uuid::new_v4()); - version1.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()); - - let mut version2 = FileMetaShallowVersion::default(); - version2.header.version_id = Some(Uuid::new_v4()); - version2.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()); - - let versions = vec![ - vec![version1.clone(), version2.clone()], - vec![version1.clone()], - vec![version2.clone()], - ]; - - let merged = merge_file_meta_versions(2, false, 10, &versions); - - // Should return versions that appear in at least quorum (2) sources - assert!(!merged.is_empty()); - } -} - -#[tokio::test] -async fn test_read_xl_meta_no_data() { - use tokio::fs; - use tokio::fs::File; - use tokio::io::AsyncWriteExt; - - let mut fm = FileMeta::new(); - - let (m, n) = (3, 2); - - for i in 0..5 { - let mut fi = FileInfo::new(i.to_string().as_str(), m, n); - fi.mod_time = Some(OffsetDateTime::now_utc()); - - fm.add_version(fi).unwrap(); - } - - // Use marshal_msg to create properly formatted data with XL headers - let buff = fm.marshal_msg().unwrap(); - - let filepath = "./test_xl.meta"; - - let mut file = File::create(filepath).await.unwrap(); - file.write_all(&buff).await.unwrap(); - - let mut f = File::open(filepath).await.unwrap(); - - let stat = f.metadata().await.unwrap(); - - let data = read_xl_meta_no_data(&mut f, stat.len() as usize).await.unwrap(); - - let mut newfm = FileMeta::default(); - newfm.unmarshal_msg(&data).unwrap(); - - fs::remove_file(filepath).await.unwrap(); - - assert_eq!(fm, newfm) -} - -#[tokio::test] -async fn test_get_file_info() { - // Test get_file_info function - let mut fm = FileMeta::new(); - let version_id = Uuid::new_v4(); - - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(version_id); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - let encoded = fm.marshal_msg().unwrap(); - - let opts = FileInfoOpts { data: false }; - let result = get_file_info(&encoded, "test-volume", "test-path", &version_id.to_string(), opts).await; - - assert!(result.is_ok()); - let file_info = result.unwrap(); - assert_eq!(file_info.volume, "test-volume"); - assert_eq!(file_info.name, "test-path"); -} - -#[tokio::test] -async fn test_file_info_from_raw() { - // Test file_info_from_raw function - let mut fm = FileMeta::new(); - let mut fi = FileInfo::new("test", 4, 2); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - let encoded = fm.marshal_msg().unwrap(); - - let raw_info = RawFileInfo { buf: encoded }; - - let result = file_info_from_raw(raw_info, "test-bucket", "test-object", false).await; - assert!(result.is_ok()); - - let file_info = result.unwrap(); - assert_eq!(file_info.volume, "test-bucket"); - assert_eq!(file_info.name, "test-object"); -} - -// Additional comprehensive tests for better coverage - -#[test] -fn test_file_meta_load_function() { - // Test FileMeta::load function - let mut fm = FileMeta::new(); - let mut fi = FileInfo::new("test", 4, 2); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - let encoded = fm.marshal_msg().unwrap(); - - // Test successful load - let loaded_fm = FileMeta::load(&encoded); - assert!(loaded_fm.is_ok()); - assert_eq!(loaded_fm.unwrap(), fm); - - // Test load with invalid data - let invalid_data = vec![0u8; 10]; - let result = FileMeta::load(&invalid_data); - assert!(result.is_err()); -} - -#[test] -fn test_file_meta_read_bytes_header() { - // Create a real FileMeta and marshal it to get proper format - let mut fm = FileMeta::new(); - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - let marshaled = fm.marshal_msg().unwrap(); - - // First call check_xl2_v1 to get the buffer after XL header validation - let (after_xl_header, _major, _minor) = FileMeta::check_xl2_v1(&marshaled).unwrap(); - - // Ensure we have at least 5 bytes for read_bytes_header - if after_xl_header.len() < 5 { - panic!("Buffer too small: {} bytes, need at least 5", after_xl_header.len()); - } - - // Now call read_bytes_header on the remaining buffer - let result = FileMeta::read_bytes_header(after_xl_header); - assert!(result.is_ok()); - let (length, remaining) = result.unwrap(); - - // The length should be greater than 0 for real data - assert!(length > 0); - // remaining should be everything after the 5-byte header - assert_eq!(remaining.len(), after_xl_header.len() - 5); - - // Test with buffer too small - let small_buf = vec![0u8; 2]; - let result = FileMeta::read_bytes_header(&small_buf); - assert!(result.is_err()); -} - -#[test] -fn test_file_meta_get_set_idx() { - let mut fm = FileMeta::new(); - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - // Test get_idx - let result = fm.get_idx(0); - assert!(result.is_ok()); - - // Test get_idx with invalid index - let result = fm.get_idx(10); - assert!(result.is_err()); - - // Test set_idx - let new_version = FileMetaVersion { - version_type: VersionType::Object, - ..Default::default() - }; - let result = fm.set_idx(0, new_version); - assert!(result.is_ok()); - - // Test set_idx with invalid index - let invalid_version = FileMetaVersion::default(); - let result = fm.set_idx(10, invalid_version); - assert!(result.is_err()); -} - -#[test] -fn test_file_meta_into_fileinfo() { - let mut fm = FileMeta::new(); - let version_id = Uuid::new_v4(); - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(version_id); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - // Test into_fileinfo with valid version_id - let result = fm.into_fileinfo("test-volume", "test-path", &version_id.to_string(), false, false); - assert!(result.is_ok()); - let file_info = result.unwrap(); - assert_eq!(file_info.volume, "test-volume"); - assert_eq!(file_info.name, "test-path"); - - // Test into_fileinfo with invalid version_id - let invalid_id = Uuid::new_v4(); - let result = fm.into_fileinfo("test-volume", "test-path", &invalid_id.to_string(), false, false); - assert!(result.is_err()); - - // Test into_fileinfo with empty version_id (should get latest) - let result = fm.into_fileinfo("test-volume", "test-path", "", false, false); - assert!(result.is_ok()); -} - -#[test] -fn test_file_meta_into_file_info_versions() { - let mut fm = FileMeta::new(); - - // Add multiple versions - for i in 0..3 { - let mut fi = FileInfo::new(&format!("test{}", i), 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000 + i).unwrap()); - fm.add_version(fi).unwrap(); - } - - let result = fm.into_file_info_versions("test-volume", "test-path", false); - assert!(result.is_ok()); - let versions = result.unwrap(); - assert_eq!(versions.versions.len(), 3); -} - -#[test] -fn test_file_meta_shallow_version_to_fileinfo() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - - let version = FileMetaVersion::from(fi.clone()); - let shallow_version = FileMetaShallowVersion::try_from(version).unwrap(); - - let result = shallow_version.to_fileinfo("test-volume", "test-path", fi.version_id, false); - assert!(result.is_ok()); - let converted_fi = result.unwrap(); - assert_eq!(converted_fi.volume, "test-volume"); - assert_eq!(converted_fi.name, "test-path"); -} - -#[test] -fn test_file_meta_version_try_from_bytes() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - let version = FileMetaVersion::from(fi); - let encoded = version.marshal_msg().unwrap(); - - // Test successful conversion - let result = FileMetaVersion::try_from(encoded.as_slice()); - assert!(result.is_ok()); - - // Test with invalid data - let invalid_data = vec![0u8; 5]; - let result = FileMetaVersion::try_from(invalid_data.as_slice()); - assert!(result.is_err()); -} - -#[test] -fn test_file_meta_version_try_from_shallow() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - let version = FileMetaVersion::from(fi); - let shallow = FileMetaShallowVersion::try_from(version.clone()).unwrap(); - - let result = FileMetaVersion::try_from(shallow); - assert!(result.is_ok()); - let converted = result.unwrap(); - assert_eq!(converted.get_version_id(), version.get_version_id()); -} - -#[test] -fn test_file_meta_version_header_from_version() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - let version = FileMetaVersion::from(fi.clone()); - - let header = FileMetaVersionHeader::from(version); - assert_eq!(header.version_id, fi.version_id); - assert_eq!(header.mod_time, fi.mod_time); -} - -#[test] -fn test_meta_object_into_fileinfo() { - let obj = MetaObject { - version_id: Some(Uuid::new_v4()), - size: 1024, - mod_time: Some(OffsetDateTime::now_utc()), - ..Default::default() - }; - - let version_id = obj.version_id; - let expected_version_id = version_id; - let file_info = obj.into_fileinfo("test-volume", "test-path", version_id, false); - assert_eq!(file_info.volume, "test-volume"); - assert_eq!(file_info.name, "test-path"); - assert_eq!(file_info.size, 1024); - assert_eq!(file_info.version_id, expected_version_id); -} - -#[test] -fn test_meta_object_from_fileinfo() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.data_dir = Some(Uuid::new_v4()); - fi.size = 2048; - fi.mod_time = Some(OffsetDateTime::now_utc()); - - let obj = MetaObject::from(fi.clone()); - assert_eq!(obj.version_id, fi.version_id); - assert_eq!(obj.data_dir, fi.data_dir); - assert_eq!(obj.size, fi.size); - assert_eq!(obj.mod_time, fi.mod_time); -} - -#[test] -fn test_meta_delete_marker_into_fileinfo() { - let marker = MetaDeleteMarker { - version_id: Some(Uuid::new_v4()), - mod_time: Some(OffsetDateTime::now_utc()), - ..Default::default() - }; - - let version_id = marker.version_id; - let expected_version_id = version_id; - let file_info = marker.into_fileinfo("test-volume", "test-path", version_id, false); - assert_eq!(file_info.volume, "test-volume"); - assert_eq!(file_info.name, "test-path"); - assert_eq!(file_info.version_id, expected_version_id); - assert!(file_info.deleted); -} - -#[test] -fn test_meta_delete_marker_from_fileinfo() { - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fi.deleted = true; - - let marker = MetaDeleteMarker::from(fi.clone()); - assert_eq!(marker.version_id, fi.version_id); - assert_eq!(marker.mod_time, fi.mod_time); -} - -#[test] -fn test_flags_enum() { - // Test Flags enum values - assert_eq!(Flags::FreeVersion as u8, 1); - assert_eq!(Flags::UsesDataDir as u8, 2); - assert_eq!(Flags::InlineData as u8, 4); -} - -#[test] -fn test_file_meta_version_header_user_data_dir() { - let header = FileMetaVersionHeader { - flags: 0, - ..Default::default() - }; - - // Test without UsesDataDir flag - assert!(!header.user_data_dir()); - - // Test with UsesDataDir flag - let header = FileMetaVersionHeader { - flags: Flags::UsesDataDir as u8, - ..Default::default() - }; - assert!(header.user_data_dir()); - - // Test with multiple flags including UsesDataDir - let header = FileMetaVersionHeader { - flags: Flags::UsesDataDir as u8 | Flags::FreeVersion as u8, - ..Default::default() - }; - assert!(header.user_data_dir()); -} - -#[test] -fn test_file_meta_version_header_ordering() { - let header1 = FileMetaVersionHeader { - mod_time: Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - let header2 = FileMetaVersionHeader { - mod_time: Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - // Test partial_cmp - assert!(header1.partial_cmp(&header2).is_some()); - - // Test cmp - header2 should be greater (newer) - use std::cmp::Ordering; - assert_eq!(header1.cmp(&header2), Ordering::Less); // header1 has earlier time - assert_eq!(header2.cmp(&header1), Ordering::Greater); // header2 has later time - assert_eq!(header1.cmp(&header1), Ordering::Equal); -} - -#[test] -fn test_merge_file_meta_versions_edge_cases() { - // Test with empty versions - let empty_versions: Vec> = vec![]; - let merged = merge_file_meta_versions(1, false, 10, &empty_versions); - assert!(merged.is_empty()); - - // Test with quorum larger than available sources - let mut version = FileMetaShallowVersion::default(); - version.header.version_id = Some(Uuid::new_v4()); - let versions = vec![vec![version]]; - let merged = merge_file_meta_versions(5, false, 10, &versions); - assert!(merged.is_empty()); - - // Test strict mode - let mut version1 = FileMetaShallowVersion::default(); - version1.header.version_id = Some(Uuid::new_v4()); - version1.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(1000).unwrap()); - - let mut version2 = FileMetaShallowVersion::default(); - version2.header.version_id = Some(Uuid::new_v4()); - version2.header.mod_time = Some(OffsetDateTime::from_unix_timestamp(2000).unwrap()); - - let versions = vec![vec![version1.clone()], vec![version2.clone()]]; - - let _merged_strict = merge_file_meta_versions(1, true, 10, &versions); - let merged_non_strict = merge_file_meta_versions(1, false, 10, &versions); - - // In strict mode, behavior might be different - assert!(!merged_non_strict.is_empty()); -} - -#[tokio::test] -async fn test_read_more_function() { - use std::io::Cursor; - - let data = b"Hello, World! This is test data."; - let mut reader = Cursor::new(data); - let mut buf = vec![0u8; 10]; - - // Test reading more data - let result = read_more(&mut reader, &mut buf, 33, 20, false).await; - assert!(result.is_ok()); - assert_eq!(buf.len(), 20); - - // Test with has_full = true and buffer already has enough data - let mut reader2 = Cursor::new(data); - let mut buf2 = vec![0u8; 5]; - let result = read_more(&mut reader2, &mut buf2, 10, 5, true).await; - assert!(result.is_ok()); - assert_eq!(buf2.len(), 5); // Should remain 5 since has >= read_size - - // Test reading beyond available data - let mut reader3 = Cursor::new(b"short"); - let mut buf3 = vec![0u8; 2]; - let result = read_more(&mut reader3, &mut buf3, 100, 98, false).await; - // Should handle gracefully even if not enough data - assert!(result.is_ok() || result.is_err()); // Either is acceptable -} - -#[tokio::test] -async fn test_read_xl_meta_no_data_edge_cases() { - use std::io::Cursor; - - // Test with empty data - let empty_data = vec![]; - let mut reader = Cursor::new(empty_data); - let result = read_xl_meta_no_data(&mut reader, 0).await; - assert!(result.is_err()); // Should fail because buffer is empty - - // Test with very small size (should fail because it's not valid XL format) - let small_data = vec![1, 2, 3]; - let mut reader = Cursor::new(small_data); - let result = read_xl_meta_no_data(&mut reader, 3).await; - assert!(result.is_err()); // Should fail because data is too small for XL format -} - -#[tokio::test] -async fn test_get_file_info_edge_cases() { - // Test with empty buffer - let empty_buf = vec![]; - let opts = FileInfoOpts { data: false }; - let result = get_file_info(&empty_buf, "volume", "path", "version", opts).await; - assert!(result.is_err()); - - // Test with invalid version_id format - let mut fm = FileMeta::new(); - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - let encoded = fm.marshal_msg().unwrap(); - - let opts = FileInfoOpts { data: false }; - let result = get_file_info(&encoded, "volume", "path", "invalid-uuid", opts).await; - assert!(result.is_err()); -} - -#[tokio::test] -async fn test_file_info_from_raw_edge_cases() { - // Test with empty buffer - let empty_raw = RawFileInfo { buf: vec![] }; - let result = file_info_from_raw(empty_raw, "bucket", "object", false).await; - assert!(result.is_err()); - - // Test with invalid buffer - let invalid_raw = RawFileInfo { - buf: vec![1, 2, 3, 4, 5], - }; - let result = file_info_from_raw(invalid_raw, "bucket", "object", false).await; - assert!(result.is_err()); -} - -#[test] -fn test_file_meta_version_invalid_cases() { - // Test invalid version - let version = FileMetaVersion { - version_type: VersionType::Invalid, - ..Default::default() - }; - assert!(!version.valid()); - - // Test version with neither object nor delete marker - let version = FileMetaVersion { - version_type: VersionType::Object, - object: None, - delete_marker: None, - ..Default::default() - }; - assert!(!version.valid()); -} - -#[test] -fn test_meta_object_edge_cases() { - let obj = MetaObject { - data_dir: None, - ..Default::default() - }; - - // Test use_data_dir with None (use_data_dir always returns true) - assert!(obj.use_data_dir()); - - // Test use_inlinedata (always returns false in current implementation) - let obj = MetaObject { - size: 128 * 1024, // 128KB threshold - ..Default::default() - }; - assert!(!obj.use_inlinedata()); // Should be false - - let obj = MetaObject { - size: 128 * 1024 - 1, - ..Default::default() - }; - assert!(!obj.use_inlinedata()); // Should also be false (always false) -} - -#[test] -fn test_file_meta_version_header_edge_cases() { - let header = FileMetaVersionHeader { - ec_n: 0, - ec_m: 0, - ..Default::default() - }; - - // Test has_ec with zero values - assert!(!header.has_ec()); - - // Test matches_not_strict with different signatures but same version_id - let version_id = Some(Uuid::new_v4()); - let header = FileMetaVersionHeader { - version_id, - version_type: VersionType::Object, - signature: [1, 2, 3, 4], - ..Default::default() - }; - let other = FileMetaVersionHeader { - version_id, - version_type: VersionType::Object, - signature: [5, 6, 7, 8], - ..Default::default() - }; - // Should match because they have same version_id and type - assert!(header.matches_not_strict(&other)); - - // Test sorts_before with same mod_time but different version_id - let time = OffsetDateTime::from_unix_timestamp(1000).unwrap(); - let header_time1 = FileMetaVersionHeader { - mod_time: Some(time), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - let header_time2 = FileMetaVersionHeader { - mod_time: Some(time), - version_id: Some(Uuid::new_v4()), - ..Default::default() - }; - - // Should use version_id for comparison when mod_time is same - let sorts_before = header_time1.sorts_before(&header_time2); - assert!(sorts_before || header_time2.sorts_before(&header_time1)); // One should sort before the other -} - -#[test] -fn test_file_meta_add_version_edge_cases() { - let mut fm = FileMeta::new(); - - // Test adding version with same version_id (should update) - let version_id = Some(Uuid::new_v4()); - let mut fi1 = FileInfo::new("test1", 4, 2); - fi1.version_id = version_id; - fi1.size = 1024; - fi1.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi1).unwrap(); - - let mut fi2 = FileInfo::new("test2", 4, 2); - fi2.version_id = version_id; - fi2.size = 2048; - fi2.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi2).unwrap(); - - // Should still have only one version, but updated - assert_eq!(fm.versions.len(), 1); - let (_, version) = fm.find_version(version_id).unwrap(); - if let Some(obj) = version.object { - assert_eq!(obj.size, 2048); // Size gets updated when adding same version_id - } -} - -#[test] -fn test_file_meta_delete_version_edge_cases() { - let mut fm = FileMeta::new(); - - // Test deleting non-existent version - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = Some(Uuid::new_v4()); - - let result = fm.delete_version(&fi); - assert!(result.is_err()); // Should fail for non-existent version -} - -#[test] -fn test_file_meta_shard_data_dir_count_edge_cases() { - let mut fm = FileMeta::new(); - - // Test with None data_dir parameter - let count = fm.shard_data_dir_count(&None, &None); - assert_eq!(count, 0); - - // Test with version_id parameter (not None) - let version_id = Some(Uuid::new_v4()); - let data_dir = Some(Uuid::new_v4()); - - let mut fi = FileInfo::new("test", 4, 2); - fi.version_id = version_id; - fi.data_dir = data_dir; - fi.mod_time = Some(OffsetDateTime::now_utc()); - fm.add_version(fi).unwrap(); - - let count = fm.shard_data_dir_count(&version_id, &data_dir); - assert_eq!(count, 0); // Should be 0 because user_data_dir() requires flag - - // Test with different version_id - let other_version_id = Some(Uuid::new_v4()); - let count = fm.shard_data_dir_count(&other_version_id, &data_dir); - assert_eq!(count, 1); // Should be 1 because the version has matching data_dir and user_data_dir() is true -} diff --git a/ecstore/src/file_meta_inline.rs b/ecstore/src/file_meta_inline.rs deleted file mode 100644 index d7c6af6f..00000000 --- a/ecstore/src/file_meta_inline.rs +++ /dev/null @@ -1,238 +0,0 @@ -use common::error::{Error, Result}; -use serde::{Deserialize, Serialize}; -use std::io::{Cursor, Read}; -use uuid::Uuid; - -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] -pub struct InlineData(Vec); - -const INLINE_DATA_VER: u8 = 1; - -impl InlineData { - pub fn new() -> Self { - Self(Vec::new()) - } - pub fn update(&mut self, buf: &[u8]) { - self.0 = buf.to_vec() - } - pub fn as_slice(&self) -> &[u8] { - self.0.as_slice() - } - pub fn version_ok(&self) -> bool { - if self.0.is_empty() { - return true; - } - - self.0[0] > 0 && self.0[0] <= INLINE_DATA_VER - } - - pub fn after_version(&self) -> &[u8] { - if self.0.is_empty() { &self.0 } else { &self.0[1..] } - } - - pub fn find(&self, key: &str) -> Result>> { - if self.0.is_empty() || !self.version_ok() { - return Ok(None); - } - - let buf = self.after_version(); - - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - while fields_len > 0 { - fields_len -= 1; - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let field = String::from_utf8(field_buff)?; - - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - cur.set_position(end as u64); - - if field.as_str() == key { - let buf = &buf[start..end]; - return Ok(Some(buf.to_vec())); - } - } - - Ok(None) - } - - pub fn validate(&self) -> Result<()> { - if self.0.is_empty() { - return Ok(()); - } - - let mut cur = Cursor::new(self.after_version()); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)?; - - while fields_len > 0 { - fields_len -= 1; - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let field = String::from_utf8(field_buff)?; - if field.is_empty() { - return Err(Error::msg("InlineData key empty")); - } - - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - cur.set_position(end as u64); - } - - Ok(()) - } - - pub fn replace(&mut self, key: &str, value: Vec) -> Result<()> { - if self.after_version().is_empty() { - let mut keys = Vec::with_capacity(1); - let mut values = Vec::with_capacity(1); - - keys.push(key.to_owned()); - values.push(value); - - return self.serialize(keys, values); - } - - let buf = self.after_version(); - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize; - let mut keys = Vec::with_capacity(fields_len + 1); - let mut values = Vec::with_capacity(fields_len + 1); - - let mut replaced = false; - - while fields_len > 0 { - fields_len -= 1; - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let find_key = String::from_utf8(field_buff)?; - - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - cur.set_position(end as u64); - - let find_value = &buf[start..end]; - - if find_key.as_str() == key { - values.push(value.clone()); - replaced = true - } else { - values.push(find_value.to_vec()); - } - - keys.push(find_key); - } - - if !replaced { - keys.push(key.to_owned()); - values.push(value); - } - - self.serialize(keys, values) - } - pub fn remove(&mut self, remove_keys: Vec) -> Result { - let buf = self.after_version(); - let mut cur = Cursor::new(buf); - - let mut fields_len = rmp::decode::read_map_len(&mut cur)? as usize; - let mut keys = Vec::with_capacity(fields_len + 1); - let mut values = Vec::with_capacity(fields_len + 1); - - let remove_key = |found_key: &str| { - for key in remove_keys.iter() { - if key.to_string().as_str() == found_key { - return true; - } - } - false - }; - - let mut found = false; - - while fields_len > 0 { - fields_len -= 1; - - let str_len = rmp::decode::read_str_len(&mut cur)?; - - let mut field_buff = vec![0u8; str_len as usize]; - - cur.read_exact(&mut field_buff)?; - - let find_key = String::from_utf8(field_buff)?; - - let bin_len = rmp::decode::read_bin_len(&mut cur)? as usize; - let start = cur.position() as usize; - let end = start + bin_len; - cur.set_position(end as u64); - - let find_value = &buf[start..end]; - - if !remove_key(&find_key) { - values.push(find_value.to_vec()); - keys.push(find_key); - } else { - found = true; - } - } - - if !found { - return Ok(false); - } - - if keys.is_empty() { - self.0 = Vec::new(); - return Ok(true); - } - - self.serialize(keys, values)?; - Ok(true) - } - fn serialize(&mut self, keys: Vec, values: Vec>) -> Result<()> { - assert_eq!(keys.len(), values.len(), "InlineData serialize: keys/values not match"); - - if keys.is_empty() { - self.0 = Vec::new(); - return Ok(()); - } - - let mut wr = Vec::new(); - - wr.push(INLINE_DATA_VER); - - let map_len = keys.len(); - - rmp::encode::write_map_len(&mut wr, map_len as u32)?; - - for i in 0..map_len { - rmp::encode::write_str(&mut wr, keys[i].as_str())?; - rmp::encode::write_bin(&mut wr, values[i].as_slice())?; - } - - self.0 = wr; - - Ok(()) - } -} diff --git a/ecstore/src/io.rs b/ecstore/src/io.rs deleted file mode 100644 index d33cb8e5..00000000 --- a/ecstore/src/io.rs +++ /dev/null @@ -1,580 +0,0 @@ -use async_trait::async_trait; -use bytes::Bytes; -use futures::TryStreamExt; -use md5::Digest; -use md5::Md5; -use pin_project_lite::pin_project; -use std::io; -use std::pin::Pin; -use std::task::Context; -use std::task::Poll; -use std::task::ready; -use tokio::io::AsyncRead; -use tokio::io::AsyncWrite; -use tokio::io::ReadBuf; -use tokio::sync::mpsc; -use tokio::sync::oneshot; -use tokio_util::io::ReaderStream; -use tokio_util::io::StreamReader; -use tracing::error; -use tracing::warn; - -// pub type FileReader = Box; -pub type FileWriter = Box; - -pub const READ_BUFFER_SIZE: usize = 1024 * 1024; - -#[derive(Debug)] -pub struct HttpFileWriter { - wd: tokio::io::DuplexStream, - err_rx: oneshot::Receiver, -} - -impl HttpFileWriter { - pub fn new(url: &str, disk: &str, volume: &str, path: &str, size: usize, append: bool) -> io::Result { - let (rd, wd) = tokio::io::duplex(READ_BUFFER_SIZE); - - let (err_tx, err_rx) = oneshot::channel::(); - - let body = reqwest::Body::wrap_stream(ReaderStream::with_capacity(rd, READ_BUFFER_SIZE)); - - let url = url.to_owned(); - let disk = disk.to_owned(); - let volume = volume.to_owned(); - let path = path.to_owned(); - - tokio::spawn(async move { - let client = reqwest::Client::new(); - if let Err(err) = client - .put(format!( - "{}/rustfs/rpc/put_file_stream?disk={}&volume={}&path={}&append={}&size={}", - url, - urlencoding::encode(&disk), - urlencoding::encode(&volume), - urlencoding::encode(&path), - append, - size - )) - .body(body) - .send() - .await - .map_err(io::Error::other) - { - error!("HttpFileWriter put file err: {:?}", err); - - if let Err(er) = err_tx.send(err) { - error!("HttpFileWriter tx.send err: {:?}", er); - } - } - }); - - Ok(Self { wd, err_rx }) - } -} - -impl AsyncWrite for HttpFileWriter { - #[tracing::instrument(level = "debug", skip(self, buf))] - fn poll_write(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &[u8]) -> Poll> { - if let Ok(err) = self.as_mut().err_rx.try_recv() { - return Poll::Ready(Err(err)); - } - - Pin::new(&mut self.wd).poll_write(cx, buf) - } - - #[tracing::instrument(level = "debug", skip(self))] - fn poll_flush(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.wd).poll_flush(cx) - } - - #[tracing::instrument(level = "debug", skip(self))] - fn poll_shutdown(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { - Pin::new(&mut self.wd).poll_shutdown(cx) - } -} - -// pub struct HttpFileReader { -// inner: FileReader, -// } - -// impl HttpFileReader { -// pub async fn new(url: &str, disk: &str, volume: &str, path: &str, offset: usize, length: usize) -> io::Result { -// let resp = reqwest::Client::new() -// .get(format!( -// "{}/rustfs/rpc/read_file_stream?disk={}&volume={}&path={}&offset={}&length={}", -// url, -// urlencoding::encode(disk), -// urlencoding::encode(volume), -// urlencoding::encode(path), -// offset, -// length -// )) -// .send() -// .await -// .map_err(io::Error::other)?; - -// let inner = Box::new(StreamReader::new(resp.bytes_stream().map_err(io::Error::other))); - -// Ok(Self { inner }) -// } -// } - -// impl AsyncRead for HttpFileReader { -// fn poll_read(mut self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { -// Pin::new(&mut self.inner).poll_read(cx, buf) -// } -// } - -#[async_trait] -pub trait Etag { - async fn etag(self) -> String; -} - -pin_project! { - #[derive(Debug)] - pub struct EtagReader { - inner: R, - bytes_tx: mpsc::Sender, - md5_rx: oneshot::Receiver, - } -} - -impl EtagReader { - pub fn new(inner: R) -> Self { - let (bytes_tx, mut bytes_rx) = mpsc::channel::(8); - let (md5_tx, md5_rx) = oneshot::channel::(); - - tokio::task::spawn_blocking(move || { - let mut md5 = Md5::new(); - while let Some(bytes) = bytes_rx.blocking_recv() { - md5.update(&bytes); - } - let digest = md5.finalize(); - let etag = hex_simd::encode_to_string(digest, hex_simd::AsciiCase::Lower); - let _ = md5_tx.send(etag); - }); - - EtagReader { inner, bytes_tx, md5_rx } - } -} - -#[async_trait] -impl Etag for EtagReader { - async fn etag(self) -> String { - drop(self.inner); - drop(self.bytes_tx); - self.md5_rx.await.unwrap() - } -} - -impl AsyncRead for EtagReader { - #[tracing::instrument(level = "info", skip_all)] - fn poll_read(self: Pin<&mut Self>, cx: &mut Context<'_>, buf: &mut ReadBuf<'_>) -> Poll> { - let me = self.project(); - - loop { - let rem = buf.remaining(); - if rem != 0 { - ready!(Pin::new(&mut *me.inner).poll_read(cx, buf))?; - if buf.remaining() == rem { - return Err(io::Error::new(io::ErrorKind::UnexpectedEof, "early eof")).into(); - } - } else { - let bytes = buf.filled(); - let bytes = Bytes::copy_from_slice(bytes); - let tx = me.bytes_tx.clone(); - tokio::spawn(async move { - if let Err(e) = tx.send(bytes).await { - warn!("EtagReader send error: {:?}", e); - } - }); - return Poll::Ready(Ok(())); - } - } - } -} - -#[cfg(test)] -mod tests { - use super::*; - use std::io::Cursor; - - #[tokio::test] - async fn test_constants() { - assert_eq!(READ_BUFFER_SIZE, 1024 * 1024); - // READ_BUFFER_SIZE is a compile-time constant, no need to assert - // assert!(READ_BUFFER_SIZE > 0); - } - - #[tokio::test] - async fn test_http_file_writer_creation() { - let writer = HttpFileWriter::new("http://localhost:8080", "test-disk", "test-volume", "test-path", 1024, false); - - assert!(writer.is_ok(), "HttpFileWriter creation should succeed"); - } - - #[tokio::test] - async fn test_http_file_writer_creation_with_special_characters() { - let writer = HttpFileWriter::new( - "http://localhost:8080", - "test disk with spaces", - "test/volume", - "test file with spaces & symbols.txt", - 1024, - false, - ); - - assert!(writer.is_ok(), "HttpFileWriter creation with special characters should succeed"); - } - - #[tokio::test] - async fn test_http_file_writer_creation_append_mode() { - let writer = HttpFileWriter::new( - "http://localhost:8080", - "test-disk", - "test-volume", - "append-test.txt", - 1024, - true, // append mode - ); - - assert!(writer.is_ok(), "HttpFileWriter creation in append mode should succeed"); - } - - #[tokio::test] - async fn test_http_file_writer_creation_zero_size() { - let writer = HttpFileWriter::new( - "http://localhost:8080", - "test-disk", - "test-volume", - "empty-file.txt", - 0, // zero size - false, - ); - - assert!(writer.is_ok(), "HttpFileWriter creation with zero size should succeed"); - } - - #[tokio::test] - async fn test_http_file_writer_creation_large_size() { - let writer = HttpFileWriter::new( - "http://localhost:8080", - "test-disk", - "test-volume", - "large-file.txt", - 1024 * 1024 * 100, // 100MB - false, - ); - - assert!(writer.is_ok(), "HttpFileWriter creation with large size should succeed"); - } - - #[tokio::test] - async fn test_http_file_writer_invalid_url() { - let writer = HttpFileWriter::new("invalid-url", "test-disk", "test-volume", "test-path", 1024, false); - - // This should still succeed at creation time, errors occur during actual I/O - assert!(writer.is_ok(), "HttpFileWriter creation should succeed even with invalid URL"); - } - - // #[tokio::test] - // async fn test_http_file_reader_creation() { - // // Test creation without actually making HTTP requests - // // We'll test the URL construction logic by checking the error messages - // let result = - // HttpFileReader::new("http://invalid-server:9999", "test-disk", "test-volume", "test-file.txt", 0, 1024).await; - - // // May succeed or fail depending on network conditions, but should not panic - // // The important thing is that the URL construction logic works - // assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic"); - // } - - // #[tokio::test] - // async fn test_http_file_reader_with_offset_and_length() { - // let result = HttpFileReader::new( - // "http://invalid-server:9999", - // "test-disk", - // "test-volume", - // "test-file.txt", - // 100, // offset - // 500, // length - // ) - // .await; - - // // May succeed or fail, but this tests parameter handling - // assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic"); - // } - - // #[tokio::test] - // async fn test_http_file_reader_zero_length() { - // let result = HttpFileReader::new( - // "http://invalid-server:9999", - // "test-disk", - // "test-volume", - // "test-file.txt", - // 0, - // 0, // zero length - // ) - // .await; - - // // May succeed or fail, but this tests zero length handling - // assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic"); - // } - - // #[tokio::test] - // async fn test_http_file_reader_with_special_characters() { - // let result = HttpFileReader::new( - // "http://invalid-server:9999", - // "test disk with spaces", - // "test/volume", - // "test file with spaces & symbols.txt", - // 0, - // 1024, - // ) - // .await; - - // // May succeed or fail, but this tests URL encoding - // assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic"); - // } - - #[tokio::test] - async fn test_etag_reader_creation() { - let data = b"hello world"; - let cursor = Cursor::new(data); - let etag_reader = EtagReader::new(cursor); - - // Test that the reader was created successfully - assert!(format!("{:?}", etag_reader).contains("EtagReader")); - } - - #[tokio::test] - async fn test_etag_reader_read_and_compute() { - let data = b"hello world"; - let cursor = Cursor::new(data); - let etag_reader = EtagReader::new(cursor); - - // Test that EtagReader can be created and the etag method works - // Note: Due to the complex implementation of EtagReader's poll_read, - // we focus on testing the creation and etag computation without reading - let etag = etag_reader.etag().await; - assert!(!etag.is_empty(), "ETag should not be empty"); - assert_eq!(etag.len(), 32, "MD5 hash should be 32 characters"); // MD5 hex string - } - - #[tokio::test] - async fn test_etag_reader_empty_data() { - let data = b""; - let cursor = Cursor::new(data); - let etag_reader = EtagReader::new(cursor); - - // Test ETag computation for empty data without reading - let etag = etag_reader.etag().await; - assert!(!etag.is_empty(), "ETag should not be empty even for empty data"); - assert_eq!(etag.len(), 32, "MD5 hash should be 32 characters"); - // MD5 of empty data should be d41d8cd98f00b204e9800998ecf8427e - assert_eq!(etag, "d41d8cd98f00b204e9800998ecf8427e", "Empty data should have known MD5"); - } - - #[tokio::test] - async fn test_etag_reader_large_data() { - let data = vec![0u8; 10000]; // 10KB of zeros - let cursor = Cursor::new(data.clone()); - let etag_reader = EtagReader::new(cursor); - - // Test ETag computation for large data without reading - let etag = etag_reader.etag().await; - assert!(!etag.is_empty(), "ETag should not be empty"); - assert_eq!(etag.len(), 32, "MD5 hash should be 32 characters"); - } - - #[tokio::test] - async fn test_etag_reader_consistent_hash() { - let data = b"test data for consistent hashing"; - - // Create two identical readers - let cursor1 = Cursor::new(data); - let etag_reader1 = EtagReader::new(cursor1); - - let cursor2 = Cursor::new(data); - let etag_reader2 = EtagReader::new(cursor2); - - // Compute ETags without reading - let etag1 = etag_reader1.etag().await; - let etag2 = etag_reader2.etag().await; - - assert_eq!(etag1, etag2, "ETags should be identical for identical data"); - } - - #[tokio::test] - async fn test_etag_reader_different_data_different_hash() { - let data1 = b"first data set"; - let data2 = b"second data set"; - - let cursor1 = Cursor::new(data1); - let etag_reader1 = EtagReader::new(cursor1); - - let cursor2 = Cursor::new(data2); - let etag_reader2 = EtagReader::new(cursor2); - - // Note: Due to the current EtagReader implementation, - // calling etag() without reading data first will return empty data hash - // This test verifies that the implementation is consistent - let etag1 = etag_reader1.etag().await; - let etag2 = etag_reader2.etag().await; - - // Both should return the same hash (empty data hash) since no data was read - assert_eq!(etag1, etag2, "ETags should be consistent when no data is read"); - assert_eq!(etag1, "d41d8cd98f00b204e9800998ecf8427e", "Should be empty data MD5"); - } - - #[tokio::test] - async fn test_etag_reader_creation_with_different_data() { - let data = b"this is a longer piece of data for testing"; - let cursor = Cursor::new(data); - let etag_reader = EtagReader::new(cursor); - - // Test ETag computation - let etag = etag_reader.etag().await; - assert!(!etag.is_empty(), "ETag should not be empty"); - assert_eq!(etag.len(), 32, "MD5 hash should be 32 characters"); - } - - // #[tokio::test] - // async fn test_file_reader_and_writer_types() { - // // Test that the type aliases are correctly defined - // let _reader: FileReader = Box::new(Cursor::new(b"test")); - // let (_writer_tx, writer_rx) = tokio::io::duplex(1024); - // let _writer: FileWriter = Box::new(writer_rx); - - // // If this compiles, the types are correctly defined - // // This is a placeholder test - remove meaningless assertion - // // assert!(true); - // } - - #[tokio::test] - async fn test_etag_trait_implementation() { - let data = b"test data for trait"; - let cursor = Cursor::new(data); - let etag_reader = EtagReader::new(cursor); - - // Test the Etag trait - let etag = etag_reader.etag().await; - assert!(!etag.is_empty(), "ETag should not be empty"); - - // Verify it's a valid hex string - assert!(etag.chars().all(|c| c.is_ascii_hexdigit()), "ETag should be a valid hex string"); - } - - #[tokio::test] - async fn test_read_buffer_size_constant() { - assert_eq!(READ_BUFFER_SIZE, 1024 * 1024); - // READ_BUFFER_SIZE is a compile-time constant, no need to assert - // assert!(READ_BUFFER_SIZE > 0); - // assert!(READ_BUFFER_SIZE % 1024 == 0, "Buffer size should be a multiple of 1024"); - } - - #[tokio::test] - async fn test_concurrent_etag_operations() { - let data1 = b"concurrent test data 1"; - let data2 = b"concurrent test data 2"; - let data3 = b"concurrent test data 3"; - - let cursor1 = Cursor::new(data1); - let cursor2 = Cursor::new(data2); - let cursor3 = Cursor::new(data3); - - let etag_reader1 = EtagReader::new(cursor1); - let etag_reader2 = EtagReader::new(cursor2); - let etag_reader3 = EtagReader::new(cursor3); - - // Compute ETags concurrently - let (result1, result2, result3) = tokio::join!(etag_reader1.etag(), etag_reader2.etag(), etag_reader3.etag()); - - // All ETags should be the same (empty data hash) since no data was read - assert_eq!(result1, result2); - assert_eq!(result2, result3); - assert_eq!(result1, result3); - - assert_eq!(result1.len(), 32); - assert_eq!(result2.len(), 32); - assert_eq!(result3.len(), 32); - - // All should be the empty data MD5 - assert_eq!(result1, "d41d8cd98f00b204e9800998ecf8427e"); - } - - // #[tokio::test] - // async fn test_edge_case_parameters() { - // // Test HttpFileWriter with edge case parameters - // let writer = HttpFileWriter::new( - // "http://localhost:8080", - // "", // empty disk - // "", // empty volume - // "", // empty path - // 0, // zero size - // false, - // ); - // assert!(writer.is_ok(), "HttpFileWriter should handle empty parameters"); - - // // Test HttpFileReader with edge case parameters - // let result = HttpFileReader::new( - // "http://invalid:9999", - // "", // empty disk - // "", // empty volume - // "", // empty path - // 0, // zero offset - // 0, // zero length - // ) - // .await; - // // May succeed or fail, but parameters should be handled - // assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic"); - // } - - // #[tokio::test] - // async fn test_url_encoding_edge_cases() { - // // Test with characters that need URL encoding - // let special_chars = "test file with spaces & symbols + % # ? = @ ! $ ( ) [ ] { } | \\ / : ; , . < > \" '"; - - // let writer = HttpFileWriter::new("http://localhost:8080", special_chars, special_chars, special_chars, 1024, false); - // assert!(writer.is_ok(), "HttpFileWriter should handle special characters"); - - // let result = HttpFileReader::new("http://invalid:9999", special_chars, special_chars, special_chars, 0, 1024).await; - // // May succeed or fail, but URL encoding should work - // assert!(result.is_ok() || result.is_err(), "HttpFileReader creation should not panic"); - // } - - #[tokio::test] - async fn test_etag_reader_with_binary_data() { - // Test with binary data including null bytes - let data = vec![0u8, 1u8, 255u8, 127u8, 128u8, 0u8, 0u8, 255u8]; - let cursor = Cursor::new(data.clone()); - let etag_reader = EtagReader::new(cursor); - - // Test ETag computation for binary data - let etag = etag_reader.etag().await; - assert!(!etag.is_empty(), "ETag should not be empty"); - assert_eq!(etag.len(), 32, "MD5 hash should be 32 characters"); - assert!(etag.chars().all(|c| c.is_ascii_hexdigit()), "ETag should be valid hex"); - } - - #[tokio::test] - async fn test_etag_reader_type_constraints() { - // Test that EtagReader works with different reader types - let data = b"type constraint test"; - - // Test with Cursor - let cursor = Cursor::new(data); - let etag_reader = EtagReader::new(cursor); - let etag = etag_reader.etag().await; - assert_eq!(etag.len(), 32); - - // Test with slice - let slice_reader = &data[..]; - let etag_reader2 = EtagReader::new(slice_reader); - let etag2 = etag_reader2.etag().await; - assert_eq!(etag2.len(), 32); - - // Both should produce the same hash for the same data - assert_eq!(etag, etag2); - } -} diff --git a/ecstore/src/metacache/mod.rs b/ecstore/src/metacache/mod.rs deleted file mode 100644 index d3baa817..00000000 --- a/ecstore/src/metacache/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub mod writer; diff --git a/ecstore/src/metacache/writer.rs b/ecstore/src/metacache/writer.rs deleted file mode 100644 index e615fe3d..00000000 --- a/ecstore/src/metacache/writer.rs +++ /dev/null @@ -1,387 +0,0 @@ -use crate::disk::MetaCacheEntry; -use crate::error::clone_err; -use common::error::{Error, Result}; -use rmp::Marker; -use std::str::from_utf8; -use tokio::io::AsyncRead; -use tokio::io::AsyncReadExt; -use tokio::io::AsyncWrite; -use tokio::io::AsyncWriteExt; -// use std::sync::Arc; -// use tokio::sync::mpsc; -// use tokio::sync::mpsc::Sender; -// use tokio::task; - -const METACACHE_STREAM_VERSION: u8 = 2; - -#[derive(Debug)] -pub struct MetacacheWriter { - wr: W, - created: bool, - // err: Option, - buf: Vec, -} - -impl MetacacheWriter { - pub fn new(wr: W) -> Self { - Self { - wr, - created: false, - // err: None, - buf: Vec::new(), - } - } - - pub async fn flush(&mut self) -> Result<()> { - self.wr.write_all(&self.buf).await?; - self.buf.clear(); - - Ok(()) - } - - pub async fn init(&mut self) -> Result<()> { - if !self.created { - rmp::encode::write_u8(&mut self.buf, METACACHE_STREAM_VERSION).map_err(|e| Error::msg(format!("{:?}", e)))?; - self.flush().await?; - self.created = true; - } - Ok(()) - } - - pub async fn write(&mut self, objs: &[MetaCacheEntry]) -> Result<()> { - if objs.is_empty() { - return Ok(()); - } - - self.init().await?; - - for obj in objs.iter() { - if obj.name.is_empty() { - return Err(Error::msg("metacacheWriter: no name")); - } - - self.write_obj(obj).await?; - } - - Ok(()) - } - - pub async fn write_obj(&mut self, obj: &MetaCacheEntry) -> Result<()> { - self.init().await?; - - rmp::encode::write_bool(&mut self.buf, true).map_err(|e| Error::msg(format!("{:?}", e)))?; - rmp::encode::write_str(&mut self.buf, &obj.name).map_err(|e| Error::msg(format!("{:?}", e)))?; - rmp::encode::write_bin(&mut self.buf, &obj.metadata).map_err(|e| Error::msg(format!("{:?}", e)))?; - self.flush().await?; - - Ok(()) - } - - // pub async fn stream(&mut self) -> Result> { - // let (sender, mut receiver) = mpsc::channel::(100); - - // let wr = Arc::new(self); - - // task::spawn(async move { - // while let Some(obj) = receiver.recv().await { - // // if obj.name.is_empty() || self.err.is_some() { - // // continue; - // // } - - // let _ = wr.write_obj(&obj); - - // // if let Err(err) = rmp::encode::write_bool(&mut self.wr, true) { - // // self.err = Some(Error::new(err)); - // // continue; - // // } - - // // if let Err(err) = rmp::encode::write_str(&mut self.wr, &obj.name) { - // // self.err = Some(Error::new(err)); - // // continue; - // // } - - // // if let Err(err) = rmp::encode::write_bin(&mut self.wr, &obj.metadata) { - // // self.err = Some(Error::new(err)); - // // continue; - // // } - // } - // }); - - // Ok(sender) - // } - - pub async fn close(&mut self) -> Result<()> { - rmp::encode::write_bool(&mut self.buf, false).map_err(|e| Error::msg(format!("{:?}", e)))?; - self.flush().await?; - Ok(()) - } -} - -pub struct MetacacheReader { - rd: R, - init: bool, - err: Option, - buf: Vec, - offset: usize, - - current: Option, -} - -impl MetacacheReader { - pub fn new(rd: R) -> Self { - Self { - rd, - init: false, - err: None, - buf: Vec::new(), - offset: 0, - current: None, - } - } - - pub async fn read_more(&mut self, read_size: usize) -> Result<&[u8]> { - let ext_size = read_size + self.offset; - - let extra = ext_size - self.offset; - if self.buf.capacity() >= ext_size { - // Extend the buffer if we have enough space. - self.buf.resize(ext_size, 0); - } else { - self.buf.extend(vec![0u8; extra]); - } - - let pref = self.offset; - - self.rd.read_exact(&mut self.buf[pref..ext_size]).await?; - - self.offset += read_size; - - let data = &self.buf[pref..ext_size]; - - Ok(data) - } - - fn reset(&mut self) { - self.buf.clear(); - self.offset = 0; - } - - async fn check_init(&mut self) -> Result<()> { - if !self.init { - let ver = match rmp::decode::read_u8(&mut self.read_more(2).await?) { - Ok(res) => res, - Err(err) => { - self.err = Some(Error::msg(format!("{:?}", err))); - 0 - } - }; - match ver { - 1 | 2 => (), - _ => { - self.err = Some(Error::msg("invalid version")); - } - } - - self.init = true; - } - Ok(()) - } - - async fn read_str_len(&mut self) -> Result { - let mark = match rmp::decode::read_marker(&mut self.read_more(1).await?) { - Ok(res) => res, - Err(err) => { - let serr = format!("{:?}", err); - self.err = Some(Error::msg(&serr)); - return Err(Error::msg(&serr)); - } - }; - - match mark { - Marker::FixStr(size) => Ok(u32::from(size)), - Marker::Str8 => Ok(u32::from(self.read_u8().await?)), - Marker::Str16 => Ok(u32::from(self.read_u16().await?)), - Marker::Str32 => Ok(self.read_u32().await?), - _marker => Err(Error::msg("str marker err")), - } - } - - async fn read_bin_len(&mut self) -> Result { - let mark = match rmp::decode::read_marker(&mut self.read_more(1).await?) { - Ok(res) => res, - Err(err) => { - let serr = format!("{:?}", err); - self.err = Some(Error::msg(&serr)); - return Err(Error::msg(&serr)); - } - }; - - match mark { - Marker::Bin8 => Ok(u32::from(self.read_u8().await?)), - Marker::Bin16 => Ok(u32::from(self.read_u16().await?)), - Marker::Bin32 => Ok(self.read_u32().await?), - _ => Err(Error::msg("bin marker err")), - } - } - - async fn read_u8(&mut self) -> Result { - let buf = self.read_more(1).await?; - - Ok(u8::from_be_bytes(buf.try_into().expect("Slice with incorrect length"))) - } - - async fn read_u16(&mut self) -> Result { - let buf = self.read_more(2).await?; - - Ok(u16::from_be_bytes(buf.try_into().expect("Slice with incorrect length"))) - } - - async fn read_u32(&mut self) -> Result { - let buf = self.read_more(4).await?; - - Ok(u32::from_be_bytes(buf.try_into().expect("Slice with incorrect length"))) - } - - pub async fn skip(&mut self, size: usize) -> Result<()> { - self.check_init().await?; - - if let Some(err) = &self.err { - return Err(clone_err(err)); - } - - let mut n = size; - - if self.current.is_some() { - n -= 1; - self.current = None; - } - - while n > 0 { - match rmp::decode::read_bool(&mut self.read_more(1).await?) { - Ok(res) => { - if !res { - return Ok(()); - } - } - Err(err) => { - let serr = format!("{:?}", err); - self.err = Some(Error::msg(&serr)); - return Err(Error::msg(&serr)); - } - }; - - let l = self.read_str_len().await?; - let _ = self.read_more(l as usize).await?; - let l = self.read_bin_len().await?; - let _ = self.read_more(l as usize).await?; - - n -= 1; - } - - Ok(()) - } - - pub async fn peek(&mut self) -> Result> { - self.check_init().await?; - - if let Some(err) = &self.err { - return Err(clone_err(err)); - } - - match rmp::decode::read_bool(&mut self.read_more(1).await?) { - Ok(res) => { - if !res { - return Ok(None); - } - } - Err(err) => { - let serr = format!("{:?}", err); - self.err = Some(Error::msg(&serr)); - return Err(Error::msg(&serr)); - } - }; - - let l = self.read_str_len().await?; - - let buf = self.read_more(l as usize).await?; - let name_buf = buf.to_vec(); - let name = match from_utf8(&name_buf) { - Ok(decoded) => decoded.to_owned(), - Err(err) => { - self.err = Some(Error::msg(err.to_string())); - return Err(Error::msg(err.to_string())); - } - }; - - let l = self.read_bin_len().await?; - - let buf = self.read_more(l as usize).await?; - - let metadata = buf.to_vec(); - - self.reset(); - - let entry = Some(MetaCacheEntry { - name, - metadata, - cached: None, - reusable: false, - }); - self.current = entry.clone(); - - Ok(entry) - } - - pub async fn read_all(&mut self) -> Result> { - let mut ret = Vec::new(); - - loop { - if let Some(entry) = self.peek().await? { - ret.push(entry); - continue; - } - - break; - } - - Ok(ret) - } -} - -#[tokio::test] -async fn test_writer() { - use std::io::Cursor; - - let mut f = Cursor::new(Vec::new()); - - let mut w = MetacacheWriter::new(&mut f); - - let mut objs = Vec::new(); - for i in 0..10 { - let info = MetaCacheEntry { - name: format!("item{}", i), - metadata: vec![0u8, 10], - cached: None, - reusable: false, - }; - println!("old {:?}", &info); - objs.push(info); - } - - w.write(&objs).await.unwrap(); - - w.close().await.unwrap(); - - let data = f.into_inner(); - - let nf = Cursor::new(data); - - let mut r = MetacacheReader::new(nf); - let nobjs = r.read_all().await.unwrap(); - - // for info in nobjs.iter() { - // println!("new {:?}", &info); - // } - - assert_eq!(objs, nobjs) -} diff --git a/ecstore/src/metrics_realtime.rs b/ecstore/src/metrics_realtime.rs index 509bc76b..91b70f50 100644 --- a/ecstore/src/metrics_realtime.rs +++ b/ecstore/src/metrics_realtime.rs @@ -3,6 +3,7 @@ use std::collections::{HashMap, HashSet}; use chrono::Utc; use common::globals::{GLOBAL_Local_Node_Name, GLOBAL_Rustfs_Addr}; use madmin::metrics::{DiskIOStats, DiskMetric, RealtimeMetrics}; +use rustfs_utils::os::get_drive_stats; use serde::{Deserialize, Serialize}; use tracing::info; @@ -14,7 +15,7 @@ use crate::{ }, new_object_layer_fn, store_api::StorageAPI, - utils::os::get_drive_stats, + // utils::os::get_drive_stats, }; #[derive(Debug, Default, Serialize, Deserialize)] diff --git a/ecstore/src/quorum.rs b/ecstore/src/quorum.rs deleted file mode 100644 index 40ad7a34..00000000 --- a/ecstore/src/quorum.rs +++ /dev/null @@ -1,268 +0,0 @@ -use crate::{disk::error::DiskError, error::clone_err}; -use common::error::Error; -use std::{collections::HashMap, fmt::Debug}; -// pub type CheckErrorFn = fn(e: &Error) -> bool; - -pub trait CheckErrorFn: Debug + Send + Sync + 'static { - fn is(&self, e: &Error) -> bool; -} - -#[derive(Debug, PartialEq, thiserror::Error)] -pub enum QuorumError { - #[error("Read quorum not met")] - Read, - #[error("disk not found")] - Write, -} - -impl QuorumError { - pub fn to_u32(&self) -> u32 { - match self { - QuorumError::Read => 0x01, - QuorumError::Write => 0x02, - } - } - - pub fn from_u32(error: u32) -> Option { - match error { - 0x01 => Some(QuorumError::Read), - 0x02 => Some(QuorumError::Write), - _ => None, - } - } -} - -pub fn base_ignored_errs() -> Vec> { - vec![ - Box::new(DiskError::DiskNotFound), - Box::new(DiskError::FaultyDisk), - Box::new(DiskError::FaultyRemoteDisk), - ] -} - -// object_op_ignored_errs -pub fn object_op_ignored_errs() -> Vec> { - let mut base = base_ignored_errs(); - - let ext: Vec> = vec![ - // Box::new(DiskError::DiskNotFound), - // Box::new(DiskError::FaultyDisk), - // Box::new(DiskError::FaultyRemoteDisk), - Box::new(DiskError::DiskAccessDenied), - Box::new(DiskError::UnformattedDisk), - Box::new(DiskError::DiskOngoingReq), - ]; - - base.extend(ext); - base -} - -// bucket_op_ignored_errs -pub fn bucket_op_ignored_errs() -> Vec> { - let mut base = base_ignored_errs(); - - let ext: Vec> = vec![Box::new(DiskError::DiskAccessDenied), Box::new(DiskError::UnformattedDisk)]; - - base.extend(ext); - base -} - -// 用于检查错误是否被忽略的函数 -fn is_err_ignored(err: &Error, ignored_errs: &[Box]) -> bool { - ignored_errs.iter().any(|ignored_err| ignored_err.is(err)) -} - -// 减少错误数量并返回出现次数最多的错误 -fn reduce_errs(errs: &[Option], ignored_errs: &[Box]) -> (usize, Option) { - let mut error_counts: HashMap = HashMap::new(); - let mut error_map: HashMap = HashMap::new(); // 存 err 位置 - let nil = "nil".to_string(); - for (i, operr) in errs.iter().enumerate() { - if let Some(err) = operr { - if is_err_ignored(err, ignored_errs) { - continue; - } - - let errstr = err.inner_string(); - - let _ = *error_map.entry(errstr.clone()).or_insert(i); - *error_counts.entry(errstr.clone()).or_insert(0) += 1; - } else { - *error_counts.entry(nil.clone()).or_insert(0) += 1; - let _ = *error_map.entry(nil.clone()).or_insert(i); - continue; - } - - // let err = operr.as_ref().unwrap(); - - // let errstr = err.to_string(); - - // let _ = *error_map.entry(errstr.clone()).or_insert(i); - // *error_counts.entry(errstr.clone()).or_insert(0) += 1; - } - - let mut max = 0; - let mut max_err = nil.clone(); - for (err, &count) in error_counts.iter() { - if count > max || (count == max && *err == nil) { - max = count; - max_err.clone_from(err); - } - } - - if let Some(&err_idx) = error_map.get(&max_err) { - let err = errs[err_idx].as_ref().map(clone_err); - (max, err) - } else if max_err == nil { - (max, None) - } else { - (0, None) - } -} - -// 根据 quorum 验证错误数量 -fn reduce_quorum_errs( - errs: &[Option], - ignored_errs: &[Box], - quorum: usize, - quorum_err: QuorumError, -) -> Option { - let (max_count, max_err) = reduce_errs(errs, ignored_errs); - if max_count >= quorum { - max_err - } else { - Some(Error::new(quorum_err)) - } -} - -// 根据读 quorum 验证错误数量 -// 返回最大错误数量的下标，或 QuorumError -pub fn reduce_read_quorum_errs( - errs: &[Option], - ignored_errs: &[Box], - read_quorum: usize, -) -> Option { - reduce_quorum_errs(errs, ignored_errs, read_quorum, QuorumError::Read) -} - -// 根据写 quorum 验证错误数量 -// 返回最大错误数量的下标，或 QuorumError -#[tracing::instrument(level = "info", skip_all)] -pub fn reduce_write_quorum_errs( - errs: &[Option], - ignored_errs: &[Box], - write_quorum: usize, -) -> Option { - reduce_quorum_errs(errs, ignored_errs, write_quorum, QuorumError::Write) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[derive(Debug)] - struct MockErrorChecker { - target_error: String, - } - - impl CheckErrorFn for MockErrorChecker { - fn is(&self, e: &Error) -> bool { - e.inner_string() == self.target_error - } - } - - fn mock_error(message: &str) -> Error { - Error::msg(message.to_string()) - } - - #[test] - fn test_reduce_errs_with_no_errors() { - let errs: Vec> = vec![]; - let ignored_errs: Vec> = vec![]; - - let (count, err) = reduce_errs(&errs, &ignored_errs); - - assert_eq!(count, 0); - assert!(err.is_none()); - } - - #[test] - fn test_reduce_errs_with_ignored_errors() { - let errs = vec![Some(mock_error("ignored_error")), Some(mock_error("ignored_error"))]; - let ignored_errs: Vec> = vec![Box::new(MockErrorChecker { - target_error: "ignored_error".to_string(), - })]; - - let (count, err) = reduce_errs(&errs, &ignored_errs); - - assert_eq!(count, 0); - assert!(err.is_none()); - } - - #[test] - fn test_reduce_errs_with_mixed_errors() { - let errs = vec![ - Some(Error::new(DiskError::FileNotFound)), - Some(Error::new(DiskError::FileNotFound)), - Some(Error::new(DiskError::FileNotFound)), - Some(Error::new(DiskError::FileNotFound)), - Some(Error::new(DiskError::FileNotFound)), - Some(Error::new(DiskError::FileNotFound)), - Some(Error::new(DiskError::FileNotFound)), - Some(Error::new(DiskError::FileNotFound)), - Some(Error::new(DiskError::FileNotFound)), - ]; - let ignored_errs: Vec> = vec![Box::new(MockErrorChecker { - target_error: "error2".to_string(), - })]; - - let (count, err) = reduce_errs(&errs, &ignored_errs); - println!("count: {}, err: {:?}", count, err); - assert_eq!(count, 9); - assert_eq!(err.unwrap().to_string(), DiskError::FileNotFound.to_string()); - } - - #[test] - fn test_reduce_errs_with_nil_errors() { - let errs = vec![None, Some(mock_error("error1")), None]; - let ignored_errs: Vec> = vec![]; - - let (count, err) = reduce_errs(&errs, &ignored_errs); - - assert_eq!(count, 2); - assert!(err.is_none()); - } - - #[test] - fn test_reduce_read_quorum_errs() { - let errs = vec![ - Some(mock_error("error1")), - Some(mock_error("error1")), - Some(mock_error("error2")), - None, - None, - ]; - let ignored_errs: Vec> = vec![]; - let read_quorum = 2; - - let result = reduce_read_quorum_errs(&errs, &ignored_errs, read_quorum); - - assert!(result.is_none()); - } - - #[test] - fn test_reduce_write_quorum_errs_with_quorum_error() { - let errs = vec![ - Some(mock_error("error1")), - Some(mock_error("error2")), - Some(mock_error("error2")), - ]; - let ignored_errs: Vec> = vec![]; - let write_quorum = 3; - - let result = reduce_write_quorum_errs(&errs, &ignored_errs, write_quorum); - - assert!(result.is_some()); - assert_eq!(result.unwrap().to_string(), QuorumError::Write.to_string()); - } -} diff --git a/ecstore/src/utils/mod.rs b/ecstore/src/utils/mod.rs index d93edd76..ed5bab69 100644 --- a/ecstore/src/utils/mod.rs +++ b/ecstore/src/utils/mod.rs @@ -4,7 +4,7 @@ pub mod ellipses; pub mod fs; pub mod hash; pub mod net; -pub mod os; +// pub mod os; pub mod path; pub mod wildcard; pub mod xml; diff --git a/ecstore/src/utils/os/linux.rs b/ecstore/src/utils/os/linux.rs deleted file mode 100644 index 2616d9d8..00000000 --- a/ecstore/src/utils/os/linux.rs +++ /dev/null @@ -1,178 +0,0 @@ -use nix::sys::stat::{self, stat}; -use nix::sys::statfs::{self, FsType, statfs}; -use std::fs::File; -use std::io::{self, BufRead, Error, ErrorKind, Result}; -use std::path::Path; - -use crate::disk::Info; - -use super::IOStats; - -/// returns total and free bytes available in a directory, e.g. `/`. -pub fn get_info(p: impl AsRef) -> std::io::Result { - let stat_fs = statfs(p.as_ref())?; - - let bsize = stat_fs.block_size() as u64; - let bfree = stat_fs.blocks_free() as u64; - let bavail = stat_fs.blocks_available() as u64; - let blocks = stat_fs.blocks() as u64; - - let reserved = match bfree.checked_sub(bavail) { - Some(reserved) => reserved, - None => { - return Err(Error::other(format!( - "detected f_bavail space ({}) > f_bfree space ({}), fs corruption at ({}). please run 'fsck'", - bavail, - bfree, - p.as_ref().display() - ))); - } - }; - - let total = match blocks.checked_sub(reserved) { - Some(total) => total * bsize, - None => { - return Err(Error::other(format!( - "detected reserved space ({}) > blocks space ({}), fs corruption at ({}). please run 'fsck'", - reserved, - blocks, - p.as_ref().display() - ))); - } - }; - - let free = bavail * bsize; - let used = match total.checked_sub(free) { - Some(used) => used, - None => { - return Err(Error::other(format!( - "detected free space ({}) > total drive space ({}), fs corruption at ({}). please run 'fsck'", - free, - total, - p.as_ref().display() - ))); - } - }; - - let st = stat(p.as_ref())?; - - Ok(Info { - total, - free, - used, - files: stat_fs.files(), - ffree: stat_fs.files_free(), - fstype: get_fs_type(stat_fs.filesystem_type()).to_string(), - - major: stat::major(st.st_dev), - minor: stat::minor(st.st_dev), - - ..Default::default() - }) -} - -/// returns the filesystem type of the underlying mounted filesystem -/// -/// TODO The following mapping could not find the corresponding constant in `nix`: -/// -/// "137d" => "EXT", -/// "4244" => "HFS", -/// "5346544e" => "NTFS", -/// "61756673" => "AUFS", -/// "ef51" => "EXT2OLD", -/// "2fc12fc1" => "zfs", -/// "ff534d42" => "cifs", -/// "53464846" => "wslfs", -fn get_fs_type(fs_type: FsType) -> &'static str { - match fs_type { - statfs::TMPFS_MAGIC => "TMPFS", - statfs::MSDOS_SUPER_MAGIC => "MSDOS", - // statfs::XFS_SUPER_MAGIC => "XFS", - statfs::NFS_SUPER_MAGIC => "NFS", - statfs::EXT4_SUPER_MAGIC => "EXT4", - statfs::ECRYPTFS_SUPER_MAGIC => "ecryptfs", - statfs::OVERLAYFS_SUPER_MAGIC => "overlayfs", - statfs::REISERFS_SUPER_MAGIC => "REISERFS", - - _ => "UNKNOWN", - } -} - -pub fn same_disk(disk1: &str, disk2: &str) -> Result { - let stat1 = stat(disk1)?; - let stat2 = stat(disk2)?; - - Ok(stat1.st_dev == stat2.st_dev) -} - -pub fn get_drive_stats(major: u32, minor: u32) -> Result { - read_drive_stats(&format!("/sys/dev/block/{}:{}/stat", major, minor)) -} - -fn read_drive_stats(stats_file: &str) -> Result { - let stats = read_stat(stats_file)?; - if stats.len() < 11 { - return Err(Error::other(format!("found invalid format while reading {}", stats_file))); - } - let mut io_stats = IOStats { - read_ios: stats[0], - read_merges: stats[1], - read_sectors: stats[2], - read_ticks: stats[3], - write_ios: stats[4], - write_merges: stats[5], - write_sectors: stats[6], - write_ticks: stats[7], - current_ios: stats[8], - total_ticks: stats[9], - req_ticks: stats[10], - ..Default::default() - }; - - if stats.len() > 14 { - io_stats.discard_ios = stats[11]; - io_stats.discard_merges = stats[12]; - io_stats.discard_sectors = stats[13]; - io_stats.discard_ticks = stats[14]; - } - Ok(io_stats) -} - -fn read_stat(file_name: &str) -> Result> { - // 打开文件 - let path = Path::new(file_name); - let file = File::open(path)?; - - // 创建一个 BufReader - let reader = io::BufReader::new(file); - - // 读取第一行 - let mut stats = Vec::new(); - if let Some(line) = reader.lines().next() { - let line = line?; - // 分割行并解析为 u64 - // https://rust-lang.github.io/rust-clippy/master/index.html#trim_split_whitespace - for token in line.split_whitespace() { - let ui64: u64 = token - .parse() - .map_err(|e| Error::new(ErrorKind::InvalidData, format!("Failed to parse token '{}': {}", token, e)))?; - stats.push(ui64); - } - } - - Ok(stats) -} - -#[cfg(test)] -mod test { - use super::get_drive_stats; - - #[ignore] // FIXME: failed in github actions - #[test] - fn test_stats() { - let major = 7; - let minor = 11; - let s = get_drive_stats(major, minor).unwrap(); - println!("{:?}", s); - } -} diff --git a/ecstore/src/utils/os/mod.rs b/ecstore/src/utils/os/mod.rs deleted file mode 100644 index 706ccc70..00000000 --- a/ecstore/src/utils/os/mod.rs +++ /dev/null @@ -1,338 +0,0 @@ -#[cfg(target_os = "linux")] -mod linux; -#[cfg(all(unix, not(target_os = "linux")))] -mod unix; -#[cfg(target_os = "windows")] -mod windows; - -#[cfg(target_os = "linux")] -pub use linux::{get_drive_stats, get_info, same_disk}; -// pub use linux::same_disk; - -#[cfg(all(unix, not(target_os = "linux")))] -pub use unix::{get_drive_stats, get_info, same_disk}; -#[cfg(target_os = "windows")] -pub use windows::{get_drive_stats, get_info, same_disk}; - -#[derive(Debug, Default, PartialEq)] -pub struct IOStats { - pub read_ios: u64, - pub read_merges: u64, - pub read_sectors: u64, - pub read_ticks: u64, - pub write_ios: u64, - pub write_merges: u64, - pub write_sectors: u64, - pub write_ticks: u64, - pub current_ios: u64, - pub total_ticks: u64, - pub req_ticks: u64, - pub discard_ios: u64, - pub discard_merges: u64, - pub discard_sectors: u64, - pub discard_ticks: u64, - pub flush_ios: u64, - pub flush_ticks: u64, -} - -#[cfg(test)] -mod tests { - use super::*; - use std::path::PathBuf; - - #[test] - fn test_get_info_valid_path() { - let temp_dir = tempfile::tempdir().unwrap(); - let info = get_info(temp_dir.path()).unwrap(); - - println!("Disk Info: {:?}", info); - - assert!(info.total > 0); - assert!(info.free > 0); - assert!(info.used > 0); - assert!(info.files > 0); - assert!(info.ffree > 0); - assert!(!info.fstype.is_empty()); - } - - #[test] - fn test_get_info_invalid_path() { - let invalid_path = PathBuf::from("/invalid/path"); - let result = get_info(&invalid_path); - - assert!(result.is_err()); - } - - #[test] - fn test_same_disk_same_path() { - let temp_dir = tempfile::tempdir().unwrap(); - let path = temp_dir.path().to_str().unwrap(); - - let result = same_disk(path, path).unwrap(); - assert!(result); - } - - #[test] - fn test_same_disk_different_paths() { - let temp_dir1 = tempfile::tempdir().unwrap(); - let temp_dir2 = tempfile::tempdir().unwrap(); - - let path1 = temp_dir1.path().to_str().unwrap(); - let path2 = temp_dir2.path().to_str().unwrap(); - - let result = same_disk(path1, path2).unwrap(); - // Note: On many systems, temporary directories are on the same disk - // This test mainly verifies the function works without error - // The actual result depends on the system configuration - println!("Same disk result for temp dirs: {}", result); - - // The function returns a boolean value as expected - let _: bool = result; // Type assertion to verify return type - } - - #[test] - fn test_get_drive_stats_default() { - let stats = get_drive_stats(0, 0).unwrap(); - assert_eq!(stats, IOStats::default()); - } - - #[test] - fn test_iostats_default_values() { - // Test that IOStats default values are all zero - let stats = IOStats::default(); - - assert_eq!(stats.read_ios, 0); - assert_eq!(stats.read_merges, 0); - assert_eq!(stats.read_sectors, 0); - assert_eq!(stats.read_ticks, 0); - assert_eq!(stats.write_ios, 0); - assert_eq!(stats.write_merges, 0); - assert_eq!(stats.write_sectors, 0); - assert_eq!(stats.write_ticks, 0); - assert_eq!(stats.current_ios, 0); - assert_eq!(stats.total_ticks, 0); - assert_eq!(stats.req_ticks, 0); - assert_eq!(stats.discard_ios, 0); - assert_eq!(stats.discard_merges, 0); - assert_eq!(stats.discard_sectors, 0); - assert_eq!(stats.discard_ticks, 0); - assert_eq!(stats.flush_ios, 0); - assert_eq!(stats.flush_ticks, 0); - } - - #[test] - fn test_iostats_equality() { - // Test IOStats equality comparison - let stats1 = IOStats::default(); - let stats2 = IOStats::default(); - assert_eq!(stats1, stats2); - - let stats3 = IOStats { - read_ios: 100, - write_ios: 50, - ..Default::default() - }; - let stats4 = IOStats { - read_ios: 100, - write_ios: 50, - ..Default::default() - }; - assert_eq!(stats3, stats4); - - // Test inequality - assert_ne!(stats1, stats3); - } - - #[test] - fn test_iostats_debug_format() { - // Test Debug trait implementation - let stats = IOStats { - read_ios: 123, - write_ios: 456, - total_ticks: 789, - ..Default::default() - }; - - let debug_str = format!("{:?}", stats); - assert!(debug_str.contains("read_ios: 123")); - assert!(debug_str.contains("write_ios: 456")); - assert!(debug_str.contains("total_ticks: 789")); - } - - #[test] - fn test_iostats_partial_eq() { - // Test PartialEq trait implementation with various field combinations - let base_stats = IOStats { - read_ios: 10, - write_ios: 20, - read_sectors: 100, - write_sectors: 200, - ..Default::default() - }; - - let same_stats = IOStats { - read_ios: 10, - write_ios: 20, - read_sectors: 100, - write_sectors: 200, - ..Default::default() - }; - - let different_read = IOStats { - read_ios: 11, // Different - write_ios: 20, - read_sectors: 100, - write_sectors: 200, - ..Default::default() - }; - - assert_eq!(base_stats, same_stats); - assert_ne!(base_stats, different_read); - } - - #[test] - fn test_get_info_path_edge_cases() { - // Test with root directory (should work on most systems) - #[cfg(unix)] - { - let result = get_info(std::path::Path::new("/")); - assert!(result.is_ok(), "Root directory should be accessible"); - - if let Ok(info) = result { - assert!(info.total > 0, "Root filesystem should have non-zero total space"); - assert!(!info.fstype.is_empty(), "Root filesystem should have a type"); - } - } - - #[cfg(windows)] - { - let result = get_info(std::path::Path::new("C:\\")); - // On Windows, C:\ might not always exist, so we don't assert success - if let Ok(info) = result { - assert!(info.total > 0); - assert!(!info.fstype.is_empty()); - } - } - } - - #[test] - fn test_get_info_nonexistent_path() { - // Test with various types of invalid paths - let invalid_paths = [ - "/this/path/definitely/does/not/exist/anywhere", - "/dev/null/invalid", // /dev/null is a file, not a directory - "", // Empty path - ]; - - for invalid_path in &invalid_paths { - let result = get_info(std::path::Path::new(invalid_path)); - assert!(result.is_err(), "Invalid path should return error: {}", invalid_path); - } - } - - #[test] - fn test_same_disk_edge_cases() { - // Test with same path (should always be true) - let temp_dir = tempfile::tempdir().unwrap(); - let path_str = temp_dir.path().to_str().unwrap(); - - let result = same_disk(path_str, path_str); - assert!(result.is_ok()); - assert!(result.unwrap(), "Same path should be on same disk"); - - // Test with parent and child directories (should be on same disk) - let child_dir = temp_dir.path().join("child"); - std::fs::create_dir(&child_dir).unwrap(); - let child_path = child_dir.to_str().unwrap(); - - let result = same_disk(path_str, child_path); - assert!(result.is_ok()); - assert!(result.unwrap(), "Parent and child should be on same disk"); - } - - #[test] - fn test_same_disk_invalid_paths() { - // Test with invalid paths - let temp_dir = tempfile::tempdir().unwrap(); - let valid_path = temp_dir.path().to_str().unwrap(); - let invalid_path = "/this/path/does/not/exist"; - - let result1 = same_disk(valid_path, invalid_path); - assert!(result1.is_err(), "Should fail with one invalid path"); - - let result2 = same_disk(invalid_path, valid_path); - assert!(result2.is_err(), "Should fail with one invalid path"); - - let result3 = same_disk(invalid_path, invalid_path); - assert!(result3.is_err(), "Should fail with both invalid paths"); - } - - #[test] - fn test_iostats_field_ranges() { - // Test that IOStats can handle large values - let large_stats = IOStats { - read_ios: u64::MAX, - write_ios: u64::MAX, - read_sectors: u64::MAX, - write_sectors: u64::MAX, - total_ticks: u64::MAX, - ..Default::default() - }; - - // Should be able to create and compare - let another_large = IOStats { - read_ios: u64::MAX, - write_ios: u64::MAX, - read_sectors: u64::MAX, - write_sectors: u64::MAX, - total_ticks: u64::MAX, - ..Default::default() - }; - - assert_eq!(large_stats, another_large); - } - - #[test] - fn test_get_drive_stats_error_handling() { - // Test with potentially invalid major/minor numbers - // Note: This might succeed on some systems, so we just ensure it doesn't panic - let result1 = get_drive_stats(999, 999); - // Don't assert success/failure as it's platform-dependent - let _ = result1; - - let result2 = get_drive_stats(u32::MAX, u32::MAX); - let _ = result2; - } - - #[cfg(unix)] - #[test] - fn test_unix_specific_paths() { - // Test Unix-specific paths - let unix_paths = ["/tmp", "/var", "/usr"]; - - for path in &unix_paths { - if std::path::Path::new(path).exists() { - let result = get_info(std::path::Path::new(path)); - if result.is_ok() { - let info = result.unwrap(); - assert!(info.total > 0, "Path {} should have non-zero total space", path); - } - } - } - } - - #[test] - fn test_iostats_clone_and_copy() { - // Test that IOStats implements Clone (if it does) - let original = IOStats { - read_ios: 42, - write_ios: 84, - ..Default::default() - }; - - // Test Debug formatting with non-default values - let debug_output = format!("{:?}", original); - assert!(debug_output.contains("42")); - assert!(debug_output.contains("84")); - } -} diff --git a/ecstore/src/utils/os/unix.rs b/ecstore/src/utils/os/unix.rs deleted file mode 100644 index 117d2f02..00000000 --- a/ecstore/src/utils/os/unix.rs +++ /dev/null @@ -1,73 +0,0 @@ -use super::IOStats; -use crate::disk::Info; -use nix::sys::{stat::stat, statfs::statfs}; -use std::io::{Error, Result}; -use std::path::Path; - -/// returns total and free bytes available in a directory, e.g. `/`. -pub fn get_info(p: impl AsRef) -> std::io::Result { - let stat = statfs(p.as_ref())?; - - let bsize = stat.block_size() as u64; - let bfree = stat.blocks_free() as u64; - let bavail = stat.blocks_available() as u64; - let blocks = stat.blocks() as u64; - - let reserved = match bfree.checked_sub(bavail) { - Some(reserved) => reserved, - None => { - return Err(Error::other(format!( - "detected f_bavail space ({}) > f_bfree space ({}), fs corruption at ({}). please run fsck", - bavail, - bfree, - p.as_ref().display() - ))); - } - }; - - let total = match blocks.checked_sub(reserved) { - Some(total) => total * bsize, - None => { - return Err(Error::other(format!( - "detected reserved space ({}) > blocks space ({}), fs corruption at ({}). please run fsck", - reserved, - blocks, - p.as_ref().display() - ))); - } - }; - - let free = bavail * bsize; - let used = match total.checked_sub(free) { - Some(used) => used, - None => { - return Err(Error::other(format!( - "detected free space ({}) > total drive space ({}), fs corruption at ({}). please run fsck", - free, - total, - p.as_ref().display() - ))); - } - }; - - Ok(Info { - total, - free, - used, - files: stat.files(), - ffree: stat.files_free(), - fstype: stat.filesystem_type_name().to_string(), - ..Default::default() - }) -} - -pub fn same_disk(disk1: &str, disk2: &str) -> Result { - let stat1 = stat(disk1)?; - let stat2 = stat(disk2)?; - - Ok(stat1.st_dev == stat2.st_dev) -} - -pub fn get_drive_stats(_major: u32, _minor: u32) -> Result { - Ok(IOStats::default()) -} diff --git a/ecstore/src/utils/os/windows.rs b/ecstore/src/utils/os/windows.rs deleted file mode 100644 index 94627150..00000000 --- a/ecstore/src/utils/os/windows.rs +++ /dev/null @@ -1,144 +0,0 @@ -#![allow(unsafe_code)] // TODO: audit unsafe code - -use super::IOStats; -use crate::disk::Info; -use std::io::{Error, ErrorKind, Result}; -use std::mem; -use std::os::windows::ffi::OsStrExt; -use std::path::Path; -use winapi::shared::minwindef::{DWORD, MAX_PATH}; -use winapi::shared::ntdef::ULARGE_INTEGER; -use winapi::um::fileapi::{GetDiskFreeSpaceExW, GetDiskFreeSpaceW, GetVolumeInformationW, GetVolumePathNameW}; -use winapi::um::winnt::{LPCWSTR, WCHAR}; - -/// returns total and free bytes available in a directory, e.g. `C:\`. -pub fn get_info(p: impl AsRef) -> Result { - let path_wide: Vec = p - .as_ref() - .canonicalize()? - .into_os_string() - .encode_wide() - .chain(std::iter::once(0)) // Null-terminate the string - .collect(); - - let mut lp_free_bytes_available: ULARGE_INTEGER = unsafe { mem::zeroed() }; - let mut lp_total_number_of_bytes: ULARGE_INTEGER = unsafe { mem::zeroed() }; - let mut lp_total_number_of_free_bytes: ULARGE_INTEGER = unsafe { mem::zeroed() }; - - let success = unsafe { - GetDiskFreeSpaceExW( - path_wide.as_ptr(), - &mut lp_free_bytes_available, - &mut lp_total_number_of_bytes, - &mut lp_total_number_of_free_bytes, - ) - }; - if success == 0 { - return Err(Error::last_os_error().into()); - } - - let total = unsafe { *lp_total_number_of_bytes.QuadPart() }; - let free = unsafe { *lp_total_number_of_free_bytes.QuadPart() }; - - if free > total { - return Err(Error::new( - ErrorKind::Other, - format!( - "detected free space ({}) > total drive space ({}), fs corruption at ({}). please run 'fsck'", - free, - total, - p.as_ref().display() - ), - ) - .into()); - } - - let mut lp_sectors_per_cluster: DWORD = 0; - let mut lp_bytes_per_sector: DWORD = 0; - let mut lp_number_of_free_clusters: DWORD = 0; - let mut lp_total_number_of_clusters: DWORD = 0; - - let success = unsafe { - GetDiskFreeSpaceW( - path_wide.as_ptr(), - &mut lp_sectors_per_cluster, - &mut lp_bytes_per_sector, - &mut lp_number_of_free_clusters, - &mut lp_total_number_of_clusters, - ) - }; - if success == 0 { - return Err(Error::last_os_error().into()); - } - - Ok(Info { - total, - free, - used: total - free, - files: lp_total_number_of_clusters as u64, - ffree: lp_number_of_free_clusters as u64, - fstype: get_fs_type(&path_wide)?, - ..Default::default() - }) -} - -/// returns leading volume name. -fn get_volume_name(v: &[WCHAR]) -> Result { - let volume_name_size: DWORD = MAX_PATH as _; - let mut lp_volume_name_buffer: [WCHAR; MAX_PATH] = [0; MAX_PATH]; - - let success = unsafe { GetVolumePathNameW(v.as_ptr(), lp_volume_name_buffer.as_mut_ptr(), volume_name_size) }; - - if success == 0 { - return Err(Error::last_os_error().into()); - } - - Ok(lp_volume_name_buffer.as_ptr()) -} - -fn utf16_to_string(v: &[WCHAR]) -> String { - let len = v.iter().position(|&x| x == 0).unwrap_or(v.len()); - String::from_utf16_lossy(&v[..len]) -} - -/// returns the filesystem type of the underlying mounted filesystem -fn get_fs_type(p: &[WCHAR]) -> Result { - let path = get_volume_name(p)?; - - let volume_name_size: DWORD = MAX_PATH as _; - let n_file_system_name_size: DWORD = MAX_PATH as _; - - let mut lp_volume_serial_number: DWORD = 0; - let mut lp_maximum_component_length: DWORD = 0; - let mut lp_file_system_flags: DWORD = 0; - - let mut lp_volume_name_buffer: [WCHAR; MAX_PATH] = [0; MAX_PATH]; - let mut lp_file_system_name_buffer: [WCHAR; MAX_PATH] = [0; MAX_PATH]; - - let success = unsafe { - GetVolumeInformationW( - path, - lp_volume_name_buffer.as_mut_ptr(), - volume_name_size, - &mut lp_volume_serial_number, - &mut lp_maximum_component_length, - &mut lp_file_system_flags, - lp_file_system_name_buffer.as_mut_ptr(), - n_file_system_name_size, - ) - }; - - if success == 0 { - return Err(Error::last_os_error().into()); - } - - Ok(utf16_to_string(&lp_file_system_name_buffer)) -} - -pub fn same_disk(_add_extensiondisk1: &str, _disk2: &str) -> Result { - Ok(false) -} - -pub fn get_drive_stats(_major: u32, _minor: u32) -> Result { - Ok(IOStats::default()) -}