From 8d1fb577e224949e235efbefc0cdbb3be9413029 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 14:07:57 +0800 Subject: [PATCH 01/10] add e2e test ci --- .github/workflows/e2e.yml | 76 ++++++++++++++++++++++++++++++++++++++ .github/workflows/rust.yml | 8 ++-- 2 files changed, 81 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/e2e.yml diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml new file mode 100644 index 00000000..f92ce3c2 --- /dev/null +++ b/.github/workflows/e2e.yml @@ -0,0 +1,76 @@ +name: e2e + +on: + push: + pull_request: + branches: [ "main" ] + +env: + CARGO_TERM_COLOR: always + +jobs: + build: + timeout-minutes: 10 + runs-on: ubuntu-latest + strategy: + matrix: + rust: + - stable + - beta + - nightly + + steps: + - name: cache protoc bin + id: cache-protoc-action + uses: actions/cache@v3 + env: + cache-name: cache-protoc-action-bin + with: + path: /usr/local/bin/protoc + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 + + - name: install protoc + if: steps.cache-protoc-action.outputs.cache-hit != 'true' + run: | + wget https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip + unzip protoc-27.0-linux-x86_64.zip -d protoc3 + mv protoc3/bin/* /usr/local/bin/ + chmod +x /usr/local/bin/protoc + rm -rf protoc-27.0-linux-x86_64.zip protoc3 + + - name: print protoc version + run: protoc --version + + - name: cache flatc bin + id: cache-flatc-action + uses: actions/cache@v3 + env: + cache-name: cache-flatc-action-bin + with: + path: /usr/local/bin/flatc + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 + + - name: install flatc + if: steps.cache-flatc-action.outputs.cache-hit != 'true' + run: | + wget https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip + unzip Linux.flatc.binary.g++-13.zip + mv flatc /usr/local/bin/ + chmod +x /usr/local/bin/flatc + rm -rf Linux.flatc.binary.g++-13.zip + + - name: checkout + uses: actions/checkout@v2 + + - name: run fs + working-directory: packages/api + run: | + cargo run & + env: + PORT: 9000 + + - name: e2e test + uses: actions-rs/cargo@v1 + with: + command: test + args: -p e2e_test --lib diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 6dd5d702..a46b7217 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -68,11 +68,13 @@ jobs: override: true components: rustfmt, clippy - - uses: actions-rs/cargo@v1 + - name: cargo build + uses: actions-rs/cargo@v1 with: command: build - - uses: actions-rs/cargo@v1 + - name: cargo test + uses: actions-rs/cargo@v1 with: command: test - args: --all + args: --all --exclude e2e_test From 33c3ceb3614ff5890a092c658c9b4c8482dcffc1 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 14:12:17 +0800 Subject: [PATCH 02/10] add e2e test ci --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index f92ce3c2..20cc1c62 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -63,7 +63,7 @@ jobs: uses: actions/checkout@v2 - name: run fs - working-directory: packages/api + working-directory: . run: | cargo run & env: From 1f40e97dd83cdc05794c4d7eaa8bdcfab0160197 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 14:33:41 +0800 Subject: [PATCH 03/10] add e2e test ci --- .github/workflows/e2e.yml | 2 +- Makefile | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 20cc1c62..6595cc8d 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -65,7 +65,7 @@ jobs: - name: run fs working-directory: . run: | - cargo run & + make e2e-server & env: PORT: 9000 diff --git a/Makefile b/Makefile index 6f05e92b..5c69f4a6 100644 --- a/Makefile +++ b/Makefile @@ -21,3 +21,7 @@ start: .PHONY: stop stop: $(DOCKER_CLI) stop $(CONTAINER_NAME) + +.PHONY: e2e +e2e-server: + sh $(shell pwd)/scripts/run.sh From d0f52a2b782b878eebfd9f02d3455ecc68d235d8 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 14:50:12 +0800 Subject: [PATCH 04/10] add cargo cache --- .github/workflows/e2e.yml | 8 ++++++++ .github/workflows/rust.yml | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 6595cc8d..3c5836f3 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -59,6 +59,14 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip + - name: cache cargo + uses: actions/cache@v3 + env: + cache-name: cache-cargo + with: + path: ~/.cargo + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 + - name: checkout uses: actions/checkout@v2 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index a46b7217..8083e47d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -59,6 +59,14 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip + - name: cache cargo + uses: actions/cache@v3 + env: + cache-name: cache-cargo + with: + path: ~/.cargo + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 + - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 From ed85e4e24cbbb68a1f760e2a695de5b3e48e2e3b Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 15:00:59 +0800 Subject: [PATCH 05/10] add probe --- .github/workflows/e2e.yml | 3 ++- Makefile | 8 ++++++-- scripts/probe.sh | 12 ++++++++++++ 3 files changed, 20 insertions(+), 3 deletions(-) create mode 100644 scripts/probe.sh diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3c5836f3..00b784ba 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -73,7 +73,8 @@ jobs: - name: run fs working-directory: . run: | - make e2e-server & + make e2e-server > /dev/null & + make probe-e2e env: PORT: 9000 diff --git a/Makefile b/Makefile index 5c69f4a6..131a4f1c 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,10 @@ start: stop: $(DOCKER_CLI) stop $(CONTAINER_NAME) -.PHONY: e2e +.PHONY: e2e-server e2e-server: - sh $(shell pwd)/scripts/run.sh + sh $(shell pwd)/scripts/run.sh + +.PHONY: probe-e2e +probe-e2e: + sh $(shell pwd)/scripts/probe.sh diff --git a/scripts/probe.sh b/scripts/probe.sh new file mode 100644 index 00000000..c85de2f8 --- /dev/null +++ b/scripts/probe.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +IP=127.0.0.1 +PORT=9000 + +while true; do + nc -zv ${IP} ${PORT} + if [[ "$?" == "0" ]]; then + exit 0 + fi + sleep 2 +done From 180905ecba43d2e8b29240fd00052b4a4054e073 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 15:09:31 +0800 Subject: [PATCH 06/10] POSIX Shell --- scripts/probe.sh | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/scripts/probe.sh b/scripts/probe.sh index c85de2f8..150ca636 100644 --- a/scripts/probe.sh +++ b/scripts/probe.sh @@ -1,11 +1,14 @@ -#!/bin/bash +#!/bin/sh + +# Please use POSIX Shell +# https://www.grymoire.com/Unix/Sh.html IP=127.0.0.1 PORT=9000 while true; do nc -zv ${IP} ${PORT} - if [[ "$?" == "0" ]]; then + if [ "$?" == "0" ]; then exit 0 fi sleep 2 From 2252ac959209a2b1ada77ead9e263535676e77d7 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 15:12:16 +0800 Subject: [PATCH 07/10] POSIX Shell --- scripts/probe.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/probe.sh b/scripts/probe.sh index 150ca636..2cda9591 100644 --- a/scripts/probe.sh +++ b/scripts/probe.sh @@ -8,7 +8,7 @@ PORT=9000 while true; do nc -zv ${IP} ${PORT} - if [ "$?" == "0" ]; then + if [ "$?" -eq "0" ]; then exit 0 fi sleep 2 From b54458c4332a3d470e20b6cbed1527a52644c1f6 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 15:20:04 +0800 Subject: [PATCH 08/10] fix cargo cache --- .github/workflows/e2e.yml | 16 ++++++++-------- .github/workflows/rust.yml | 16 ++++++++-------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 00b784ba..3a900a7a 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -59,14 +59,6 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip - - name: cache cargo - uses: actions/cache@v3 - env: - cache-name: cache-cargo - with: - path: ~/.cargo - key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 - - name: checkout uses: actions/checkout@v2 @@ -83,3 +75,11 @@ jobs: with: command: test args: -p e2e_test --lib + + - name: cache cargo + uses: actions/cache@v3 + env: + cache-name: cache-cargo + with: + path: ~/.cargo + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8083e47d..00eb6f8b 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -59,14 +59,6 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip - - name: cache cargo - uses: actions/cache@v3 - env: - cache-name: cache-cargo - with: - path: ~/.cargo - key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 - - uses: actions/checkout@v2 - uses: actions-rs/toolchain@v1 @@ -86,3 +78,11 @@ jobs: with: command: test args: --all --exclude e2e_test + + - name: cache cargo + uses: actions/cache@v3 + env: + cache-name: cache-cargo + with: + path: ~/.cargo + key: ${{ runner.os }}-build-${{ env.cache-name }}-v0.0.1 From 0fa13379e4f230b17bc1dd44bb07366b5a483486 Mon Sep 17 00:00:00 2001 From: JimChenWYU Date: Tue, 24 Sep 2024 16:17:31 +0800 Subject: [PATCH 09/10] fix --- .github/workflows/e2e.yml | 14 +++++++------- .github/workflows/rust.yml | 23 ++++++++--------------- 2 files changed, 15 insertions(+), 22 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3a900a7a..0b15d245 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,7 +16,6 @@ jobs: matrix: rust: - stable - - beta - nightly steps: @@ -59,6 +58,12 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip + - name: toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ matrix.rust }} + # components: rustfmt, clippy + - name: checkout uses: actions/checkout@v2 @@ -67,14 +72,9 @@ jobs: run: | make e2e-server > /dev/null & make probe-e2e - env: - PORT: 9000 - name: e2e test - uses: actions-rs/cargo@v1 - with: - command: test - args: -p e2e_test --lib + run: cargo test -p e2e_test --lib - name: cache cargo uses: actions/cache@v3 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 00eb6f8b..7f86763d 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -16,7 +16,6 @@ jobs: matrix: rust: - stable - - beta - nightly steps: @@ -59,25 +58,19 @@ jobs: chmod +x /usr/local/bin/flatc rm -rf Linux.flatc.binary.g++-13.zip + - name: toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: ${{ matrix.rust }} + # components: rustfmt, clippy + - uses: actions/checkout@v2 - - uses: actions-rs/toolchain@v1 - with: - profile: minimal - toolchain: ${{ matrix.rust }} - override: true - components: rustfmt, clippy - - name: cargo build - uses: actions-rs/cargo@v1 - with: - command: build + run: cargo build - name: cargo test - uses: actions-rs/cargo@v1 - with: - command: test - args: --all --exclude e2e_test + run: cargo test --all --exclude e2e_test - name: cache cargo uses: actions/cache@v3 From 72201049f0f0854e61361fb1f81abbe874c5e4b0 Mon Sep 17 00:00:00 2001 From: weisd Date: Wed, 25 Sep 2024 16:54:48 +0800 Subject: [PATCH 10/10] fix: delete_objects --- ecstore/src/sets.rs | 53 +++++++++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 19 deletions(-) diff --git a/ecstore/src/sets.rs b/ecstore/src/sets.rs index c0754384..81aaefca 100644 --- a/ecstore/src/sets.rs +++ b/ecstore/src/sets.rs @@ -228,31 +228,46 @@ impl StorageAPI for Sets { } } - let semaphore = Arc::new(Semaphore::new(num_cpus::get())); - let mut jhs = Vec::with_capacity(semaphore.available_permits()); + // let semaphore = Arc::new(Semaphore::new(num_cpus::get())); + // let mut jhs = Vec::with_capacity(semaphore.available_permits()); + // for (k, v) in set_obj_map { + // let disks = self.get_disks(k); + // let semaphore = semaphore.clone(); + // let opts = opts.clone(); + // let bucket = bucket.to_string(); + + // let jh = tokio::spawn(async move { + // let _permit = semaphore.acquire().await.unwrap(); + // let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); + // disks.delete_objects(&bucket, objs, opts).await + // }); + // jhs.push(jh); + // } + + // let mut results = Vec::with_capacity(jhs.len()); + // for jh in jhs { + // results.push(jh.await?.unwrap()); + // } + + // for (dobjects, errs) in results { + // del_objects.extend(dobjects); + // del_errs.extend(errs); + // } + + // TODO: 并发 for (k, v) in set_obj_map { let disks = self.get_disks(k); - let semaphore = semaphore.clone(); - let opts = opts.clone(); - let bucket = bucket.to_string(); + let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); + let (dobjects, errs) = disks.delete_objects(bucket, objs, opts.clone()).await?; - let jh = tokio::spawn(async move { - let _permit = semaphore.acquire().await.unwrap(); - let objs: Vec = v.iter().map(|v| v.obj.clone()).collect(); - disks.delete_objects(&bucket, objs, opts).await - }); - jhs.push(jh); - } + for (i, err) in errs.into_iter().enumerate() { + let obj = v.get(i).unwrap(); - let mut results = Vec::with_capacity(jhs.len()); - for jh in jhs { - results.push(jh.await?.unwrap()); - } + del_errs[obj.orig_idx] = err; - for (dobjects, errs) in results { - del_objects.extend(dobjects); - del_errs.extend(errs); + del_objects[obj.orig_idx] = dobjects.get(i).unwrap().clone(); + } } Ok((del_objects, del_errs))