rebase to main

Signed-off-by: junxiang Mu <1948535941@qq.com>

Cargo.lock

@@ -194,7 +194,9 @@ dependencies = [
  "s3s",
  "snafu",
  "tokio",
+ "tokio-util",
  "tracing",
+ "transform-stream",
  "url",
 ]
 

Cargo.toml

@@ -99,6 +99,7 @@ tonic = { version = "0.12.3", features = ["gzip"] }
 tonic-build = "0.12.3"
 tonic-reflection = "0.12"
 tokio-stream = "0.1.17"
+tokio-util = { version = "0.7.13", features = ["io", "compat"] }
 tower = { version = "0.5.2", features = ["timeout"] }
 tracing = "0.1.41"
 tracing-error = "0.2.1"

common/protos/src/generated/flatbuffers_generated/models.rs

@@ -1,9 +1,10 @@
 // automatically generated by the FlatBuffers compiler, do not modify
 
+
 // @generated
 
-use core::cmp::Ordering;
 use core::mem;
+use core::cmp::Ordering;
 
 extern crate flatbuffers;
 use self::flatbuffers::{EndianScalar, Follow};
@@ -11,114 +12,112 @@ use self::flatbuffers::{EndianScalar, Follow};
 #[allow(unused_imports, dead_code)]
 pub mod models {
 
-    use core::cmp::Ordering;
-    use core::mem;
+  use core::mem;
+  use core::cmp::Ordering;
 
-    extern crate flatbuffers;
-    use self::flatbuffers::{EndianScalar, Follow};
+  extern crate flatbuffers;
+  use self::flatbuffers::{EndianScalar, Follow};
 
-    pub enum PingBodyOffset {}
-    #[derive(Copy, Clone, PartialEq)]
+pub enum PingBodyOffset {}
+#[derive(Copy, Clone, PartialEq)]
 
-    pub struct PingBody<'a> {
-        pub _tab: flatbuffers::Table<'a>,
+pub struct PingBody<'a> {
+  pub _tab: flatbuffers::Table<'a>,
+}
+
+impl<'a> flatbuffers::Follow<'a> for PingBody<'a> {
+  type Inner = PingBody<'a>;
+  #[inline]
+  unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
+    Self { _tab: flatbuffers::Table::new(buf, loc) }
+  }
+}
+
+impl<'a> PingBody<'a> {
+  pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4;
+
+  pub const fn get_fully_qualified_name() -> &'static str {
+    "models.PingBody"
+  }
+
+  #[inline]
+  pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
+    PingBody { _tab: table }
+  }
+  #[allow(unused_mut)]
+  pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
+    _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
+    args: &'args PingBodyArgs<'args>
+  ) -> flatbuffers::WIPOffset<PingBody<'bldr>> {
+    let mut builder = PingBodyBuilder::new(_fbb);
+    if let Some(x) = args.payload { builder.add_payload(x); }
+    builder.finish()
+  }
+
+
+  #[inline]
+  pub fn payload(&self) -> Option<flatbuffers::Vector<'a, u8>> {
+    // Safety:
+    // Created from valid Table for this object
+    // which contains a valid value in this slot
+    unsafe { self._tab.get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(PingBody::VT_PAYLOAD, None)}
+  }
+}
+
+impl flatbuffers::Verifiable for PingBody<'_> {
+  #[inline]
+  fn run_verifier(
+    v: &mut flatbuffers::Verifier, pos: usize
+  ) -> Result<(), flatbuffers::InvalidFlatbuffer> {
+    use self::flatbuffers::Verifiable;
+    v.visit_table(pos)?
+     .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("payload", Self::VT_PAYLOAD, false)?
+     .finish();
+    Ok(())
+  }
+}
+pub struct PingBodyArgs<'a> {
+    pub payload: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
+}
+impl<'a> Default for PingBodyArgs<'a> {
+  #[inline]
+  fn default() -> Self {
+    PingBodyArgs {
+      payload: None,
     }
+  }
+}
 
-    impl<'a> flatbuffers::Follow<'a> for PingBody<'a> {
-        type Inner = PingBody<'a>;
-        #[inline]
-        unsafe fn follow(buf: &'a [u8], loc: usize) -> Self::Inner {
-            Self {
-                _tab: flatbuffers::Table::new(buf, loc),
-            }
-        }
+pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
+  fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
+  start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
+}
+impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> {
+  #[inline]
+  pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset<flatbuffers::Vector<'b , u8>>) {
+    self.fbb_.push_slot_always::<flatbuffers::WIPOffset<_>>(PingBody::VT_PAYLOAD, payload);
+  }
+  #[inline]
+  pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> {
+    let start = _fbb.start_table();
+    PingBodyBuilder {
+      fbb_: _fbb,
+      start_: start,
     }
+  }
+  #[inline]
+  pub fn finish(self) -> flatbuffers::WIPOffset<PingBody<'a>> {
+    let o = self.fbb_.end_table(self.start_);
+    flatbuffers::WIPOffset::new(o.value())
+  }
+}
 
-    impl<'a> PingBody<'a> {
-        pub const VT_PAYLOAD: flatbuffers::VOffsetT = 4;
+impl core::fmt::Debug for PingBody<'_> {
+  fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+    let mut ds = f.debug_struct("PingBody");
+      ds.field("payload", &self.payload());
+      ds.finish()
+  }
+}
+}  // pub mod models
 
-        pub const fn get_fully_qualified_name() -> &'static str {
-            "models.PingBody"
-        }
-
-        #[inline]
-        pub unsafe fn init_from_table(table: flatbuffers::Table<'a>) -> Self {
-            PingBody { _tab: table }
-        }
-        #[allow(unused_mut)]
-        pub fn create<'bldr: 'args, 'args: 'mut_bldr, 'mut_bldr, A: flatbuffers::Allocator + 'bldr>(
-            _fbb: &'mut_bldr mut flatbuffers::FlatBufferBuilder<'bldr, A>,
-            args: &'args PingBodyArgs<'args>,
-        ) -> flatbuffers::WIPOffset<PingBody<'bldr>> {
-            let mut builder = PingBodyBuilder::new(_fbb);
-            if let Some(x) = args.payload {
-                builder.add_payload(x);
-            }
-            builder.finish()
-        }
-
-        #[inline]
-        pub fn payload(&self) -> Option<flatbuffers::Vector<'a, u8>> {
-            // Safety:
-            // Created from valid Table for this object
-            // which contains a valid value in this slot
-            unsafe {
-                self._tab
-                    .get::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'a, u8>>>(PingBody::VT_PAYLOAD, None)
-            }
-        }
-    }
-
-    impl flatbuffers::Verifiable for PingBody<'_> {
-        #[inline]
-        fn run_verifier(v: &mut flatbuffers::Verifier, pos: usize) -> Result<(), flatbuffers::InvalidFlatbuffer> {
-            use self::flatbuffers::Verifiable;
-            v.visit_table(pos)?
-                .visit_field::<flatbuffers::ForwardsUOffset<flatbuffers::Vector<'_, u8>>>("payload", Self::VT_PAYLOAD, false)?
-                .finish();
-            Ok(())
-        }
-    }
-    pub struct PingBodyArgs<'a> {
-        pub payload: Option<flatbuffers::WIPOffset<flatbuffers::Vector<'a, u8>>>,
-    }
-    impl<'a> Default for PingBodyArgs<'a> {
-        #[inline]
-        fn default() -> Self {
-            PingBodyArgs { payload: None }
-        }
-    }
-
-    pub struct PingBodyBuilder<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> {
-        fbb_: &'b mut flatbuffers::FlatBufferBuilder<'a, A>,
-        start_: flatbuffers::WIPOffset<flatbuffers::TableUnfinishedWIPOffset>,
-    }
-    impl<'a: 'b, 'b, A: flatbuffers::Allocator + 'a> PingBodyBuilder<'a, 'b, A> {
-        #[inline]
-        pub fn add_payload(&mut self, payload: flatbuffers::WIPOffset<flatbuffers::Vector<'b, u8>>) {
-            self.fbb_
-                .push_slot_always::<flatbuffers::WIPOffset<_>>(PingBody::VT_PAYLOAD, payload);
-        }
-        #[inline]
-        pub fn new(_fbb: &'b mut flatbuffers::FlatBufferBuilder<'a, A>) -> PingBodyBuilder<'a, 'b, A> {
-            let start = _fbb.start_table();
-            PingBodyBuilder {
-                fbb_: _fbb,
-                start_: start,
-            }
-        }
-        #[inline]
-        pub fn finish(self) -> flatbuffers::WIPOffset<PingBody<'a>> {
-            let o = self.fbb_.end_table(self.start_);
-            flatbuffers::WIPOffset::new(o.value())
-        }
-    }
-
-    impl core::fmt::Debug for PingBody<'_> {
-        fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
-            let mut ds = f.debug_struct("PingBody");
-            ds.field("payload", &self.payload());
-            ds.finish()
-        }
-    }
-} // pub mod models

rustfs/Cargo.toml

@@ -47,7 +47,7 @@ serde.workspace = true
 serde_json.workspace = true
 tracing.workspace = true
 time = { workspace = true, features = ["parsing", "formatting", "serde"] }
-tokio-util = { version = "0.7.13", features = ["io", "compat"] }
+tokio-util.workspace = true
 tokio = { workspace = true, features = [
     "rt-multi-thread",
     "macros",

rustfs/src/storage/ecfs.rs

@@ -45,8 +45,6 @@ use ecstore::xhttp;
 use futures::pin_mut;
 use futures::{Stream, StreamExt};
 use http::HeaderMap;
-use iam::policy::action::Action;
-use iam::policy::action::S3Action;
 use lazy_static::lazy_static;
 use log::warn;
 use policy::auth;

s3select/api/Cargo.toml

@@ -16,5 +16,7 @@ object_store = "0.11.2"
 s3s.workspace = true
 snafu = { workspace = true, features = ["backtrace"] }
 tokio.workspace = true
+tokio-util.workspace = true
 tracing.workspace = true
+transform-stream.workspace = true
 url.workspace = true

s3select/api/src/object_store.rs

@@ -1,13 +1,14 @@
 use async_trait::async_trait;
 use bytes::Bytes;
 use chrono::Utc;
+use ecstore::io::READ_BUFFER_SIZE;
 use ecstore::new_object_layer_fn;
 use ecstore::store::ECStore;
 use ecstore::store_api::ObjectIO;
 use ecstore::store_api::ObjectOptions;
 use ecstore::StorageAPI;
-use futures::stream;
-use futures::StreamExt;
+use futures::pin_mut;
+use futures::{Stream, StreamExt};
 use futures_core::stream::BoxStream;
 use http::HeaderMap;
 use object_store::path::Path;
@@ -28,7 +29,9 @@ use s3s::s3_error;
 use s3s::S3Result;
 use std::ops::Range;
 use std::sync::Arc;
+use tokio_util::io::ReaderStream;
 use tracing::info;
+use transform_stream::AsyncTryStream;
 
 #[derive(Debug)]
 pub struct EcObjectStore {
@@ -76,16 +79,16 @@ impl ObjectStore for EcObjectStore {
                 source: "can not get object info".into(),
             })?;
 
-        let stream = stream::unfold(reader.stream, |mut blob| async move {
-            match blob.next().await {
-                Some(Ok(chunk)) => {
-                    let bytes = chunk;
-                    Some((Ok(bytes), blob))
-                }
-                _ => None,
-            }
-        })
-        .boxed();
+        // let stream = stream::unfold(reader.stream, |mut blob| async move {
+        //     match blob.next().await {
+        //         Some(Ok(chunk)) => {
+        //             let bytes = chunk;
+        //             Some((Ok(bytes), blob))
+        //         }
+        //         _ => None,
+        //     }
+        // })
+        // .boxed();
         let meta = ObjectMeta {
             location: location.clone(),
             last_modified: Utc::now(),
@@ -96,7 +99,9 @@ impl ObjectStore for EcObjectStore {
         let attributes = Attributes::default();
 
         Ok(GetResult {
-            payload: object_store::GetResultPayload::Stream(stream),
+            payload: object_store::GetResultPayload::Stream(
+                bytes_stream(ReaderStream::with_capacity(reader.stream, READ_BUFFER_SIZE), reader.object_info.size).boxed(),
+            ),
             meta,
             range: 0..reader.object_info.size,
             attributes,
@@ -148,3 +153,25 @@ impl ObjectStore for EcObjectStore {
         unimplemented!()
     }
 }
+
+pub fn bytes_stream<S>(stream: S, content_length: usize) -> impl Stream<Item = Result<Bytes>> + Send + 'static
+where
+    S: Stream<Item = Result<Bytes, std::io::Error>> + Send + 'static,
+{
+    AsyncTryStream::<Bytes, o_Error, _>::new(|mut y| async move {
+        pin_mut!(stream);
+        let mut remaining: usize = content_length;
+        while let Some(result) = stream.next().await {
+            let mut bytes = result.map_err(|e| o_Error::Generic {
+                store: "",
+                source: Box::new(e),
+            })?;
+            if bytes.len() > remaining {
+                bytes.truncate(remaining);
+            }
+            remaining -= bytes.len();
+            y.yield_ok(bytes).await;
+        }
+        Ok(())
+    })
+}