mirror of
https://github.com/rustfs/rustfs.git
synced 2026-01-17 01:30:33 +00:00
Merge branch 'main' of github.com:rustfs/s3-rustfs into feature/logger
# Conflicts: # Cargo.lock # Cargo.toml # rustfs/src/main.rs
This commit is contained in:
2
.cargo/config.toml
Normal file
2
.cargo/config.toml
Normal file
@@ -0,0 +1,2 @@
|
||||
[target.x86_64-unknown-linux-gnu]
|
||||
rustflags = ["-Clink-arg=-fuse-ld=lld"]
|
||||
1
.github/actions/setup/action.yml
vendored
1
.github/actions/setup/action.yml
vendored
@@ -21,6 +21,7 @@ runs:
|
||||
run: |
|
||||
sudo apt update
|
||||
sudo apt install -y \
|
||||
lld \
|
||||
libdbus-1-dev \
|
||||
libwayland-dev \
|
||||
libwebkit2gtk-4.1-dev \
|
||||
|
||||
138
.github/workflows/build.yml
vendored
138
.github/workflows/build.yml
vendored
@@ -7,6 +7,7 @@ on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
tags: [ 'v*', '*' ]
|
||||
|
||||
jobs:
|
||||
build-rustfs:
|
||||
@@ -15,9 +16,9 @@ jobs:
|
||||
strategy:
|
||||
matrix:
|
||||
variant:
|
||||
- { profile: dev, target: x86_64-unknown-linux-gnu, glibc: "default" }
|
||||
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
|
||||
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" }
|
||||
- { profile: dev, target: x86_64-unknown-linux-gnu, glibc: "default" }
|
||||
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "default" }
|
||||
- { profile: release, target: x86_64-unknown-linux-gnu, glibc: "2.31" }
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -25,6 +26,15 @@ jobs:
|
||||
with:
|
||||
cache-shared-key: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }}
|
||||
|
||||
- name: Download and Extract Static Assets
|
||||
run: |
|
||||
url="https://dl.rustfs.com/console/rustfs-console-latest.zip"
|
||||
mkdir -p static
|
||||
curl -L -o static_assets.zip "$url"
|
||||
unzip -o static_assets.zip -d ./rustfs/static
|
||||
rm static_assets.zip
|
||||
ls -la ./rustfs/static
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
./scripts/build.py \
|
||||
@@ -32,16 +42,130 @@ jobs:
|
||||
--target ${{ matrix.variant.target }} \
|
||||
--glibc ${{ matrix.variant.glibc }}
|
||||
|
||||
- name: Package Binary and Static Assets
|
||||
id: package
|
||||
run: |
|
||||
# Create artifact filename
|
||||
ARTIFACT_NAME="rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}"
|
||||
if [ "${{ matrix.variant.glibc }}" != "default" ]; then
|
||||
ARTIFACT_NAME="${ARTIFACT_NAME}-glibc${{ matrix.variant.glibc }}"
|
||||
fi
|
||||
echo "artifact_name=${ARTIFACT_NAME}" >> $GITHUB_OUTPUT
|
||||
|
||||
# Determine binary path
|
||||
bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.bin"
|
||||
if [ -f "target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin" ]; then
|
||||
bin_path="target/artifacts/rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.glibc${{ matrix.variant.glibc }}.bin"
|
||||
fi
|
||||
|
||||
# Create package
|
||||
mkdir -p ${ARTIFACT_NAME}
|
||||
cp "$bin_path" ${ARTIFACT_NAME}/rustfs
|
||||
zip -r ${ARTIFACT_NAME}.zip ${ARTIFACT_NAME}
|
||||
ls -la
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: rustfs.${{ matrix.variant.profile }}.${{ matrix.variant.target }}.${{ matrix.variant.glibc }}
|
||||
path: ./target/artifacts/*
|
||||
name: ${{ steps.package.outputs.artifact_name }}
|
||||
path: ${{ steps.package.outputs.artifact_name }}.zip
|
||||
retention-days: 7
|
||||
|
||||
build-rustfs-gui:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-rustfs
|
||||
|
||||
strategy:
|
||||
matrix:
|
||||
variant:
|
||||
- { profile: release, target: x86_64-unknown-linux-gnu }
|
||||
# - { profile: release, target: x86_64-apple-darwin }
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: Download artifact
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
name: "rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}"
|
||||
- name: Display structure of downloaded files
|
||||
run: |
|
||||
ls -R
|
||||
unzip -o -j "rustfs-${{ matrix.variant.profile }}-${{ matrix.variant.target }}.zip" -d ./cli/rustfs-gui/embedded-rustfs/
|
||||
ls -la cli/rustfs-gui/embedded-rustfs
|
||||
- name: Cache dioxus-cli
|
||||
uses: actions/cache@v4
|
||||
with:
|
||||
path: ~/.cargo/bin/dx
|
||||
key: ${{ runner.os }}-dioxus-cli-${{ hashFiles('**/Cargo.lock') }}
|
||||
restore-keys: |
|
||||
${{ runner.os }}-dioxus-cli-
|
||||
|
||||
- name: Install dioxus-cli
|
||||
run: |
|
||||
if [ ! -f ~/.cargo/bin/dx ]; then
|
||||
cargo install dioxus-cli
|
||||
fi
|
||||
|
||||
- name: Build and Bundle rustfs-gui
|
||||
run: |
|
||||
ls -la
|
||||
|
||||
release_path="target/${{ matrix.variant.target }}"
|
||||
mkdir -p ${release_path}
|
||||
cd cli/rustfs-gui
|
||||
ls -la embedded-rustfs
|
||||
|
||||
# Configure the linker based on the target
|
||||
case "${{ matrix.target }}" in
|
||||
"x86_64-unknown-linux-gnu")
|
||||
# Default gcc
|
||||
export CC_x86_64_unknown_linux_gnu=gcc
|
||||
export CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=gcc
|
||||
;;
|
||||
"aarch64-unknown-linux-gnu")
|
||||
# AArch64 Cross-compiler
|
||||
export CC_aarch64_unknown_linux_gnu=aarch64-linux-gnu-gcc
|
||||
export CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc
|
||||
;;
|
||||
"x86_64-apple-darwin")
|
||||
# macOS default clang
|
||||
export CC_x86_64_apple_darwin=clang
|
||||
export CARGO_TARGET_X86_64_APPLE_DARWIN_LINKER=clang
|
||||
;;
|
||||
"aarch64-apple-darwin")
|
||||
# macOS ARM64 used clang
|
||||
export CC_aarch64_apple_darwin=clang
|
||||
export CARGO_TARGET_AARCH64_APPLE_DARWIN_LINKER=clang
|
||||
;;
|
||||
esac
|
||||
# Validating Environment Variables (for Debugging)
|
||||
echo "CC for ${{ matrix.target }}: $CC_${{ matrix.target }}"
|
||||
echo "Linker for ${{ matrix.target }}: $CARGO_TARGET_${{ matrix.target }}_LINKER"
|
||||
|
||||
if [[ "${{ matrix.variant.target }}" == *"apple-darwin"* ]]; then
|
||||
dx bundle --platform macos --package-types "macos" --package-types "dmg" --package-types "ios" --release --profile release --out-dir ../../${release_path}
|
||||
elif [[ "${{ matrix.variant.target }}" == *"windows-msvc"* ]]; then
|
||||
dx bundle --platform windows --package-types "msi" --release --profile release --out-dir ../../${release_path}
|
||||
elif [[ "${{ matrix.variant.target }}" == *"unknown-linux-gnu"* ]]; then
|
||||
dx bundle --platform linux --package-types "deb" --package-types "rpm" --package-types "appimage" --release --profile release --out-dir ../../${release_path}
|
||||
fi
|
||||
cd ../..
|
||||
GUI_ARTIFACT_NAME="rustfs-gui-${{ matrix.variant.profile }}-${{ matrix.variant.target }}"
|
||||
zip -r ${GUI_ARTIFACT_NAME}.zip ${release_path}/*
|
||||
echo "gui_artifact_name=${GUI_ARTIFACT_NAME}" >> $GITHUB_OUTPUT
|
||||
ls -la ${release_path}
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: ${{ steps.package.outputs.gui_artifact_name }}
|
||||
path: ${{ steps.package.outputs.gui_artifact_name }}.zip
|
||||
retention-days: 7
|
||||
|
||||
merge:
|
||||
runs-on: ubuntu-latest
|
||||
needs: build-rustfs
|
||||
needs: [ build-rustfs, build-rustfs-gui ]
|
||||
steps:
|
||||
- uses: actions/upload-artifact/merge@v4
|
||||
with:
|
||||
name: rustfs
|
||||
name: rustfs-packages
|
||||
pattern: 'rustfs-*'
|
||||
delete-merged: true
|
||||
|
||||
96
.github/workflows/ci.yml
vendored
96
.github/workflows/ci.yml
vendored
@@ -37,13 +37,6 @@ jobs:
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup
|
||||
with:
|
||||
cache-shared-key: "develop"
|
||||
|
||||
- name: Install s3s-e2e
|
||||
run: |
|
||||
cargo install s3s-e2e --git https://github.com/Nugine/s3s.git
|
||||
s3s-e2e --version
|
||||
|
||||
- name: Format
|
||||
run: cargo fmt --all --check
|
||||
@@ -52,41 +45,24 @@ jobs:
|
||||
run: cargo check --all-targets
|
||||
# TODO: cargo clippy
|
||||
|
||||
- name: Test
|
||||
run: cargo test --all --exclude e2e_test
|
||||
|
||||
- name: Build debug
|
||||
run: |
|
||||
touch rustfs/build.rs
|
||||
cargo build -p rustfs --bins
|
||||
|
||||
# - name: Build release
|
||||
# run: |
|
||||
# touch rustfs/build.rs
|
||||
# cargo build -p rustfs --bins --release
|
||||
|
||||
- run: |
|
||||
- name: Pack artifacts
|
||||
run: |
|
||||
mkdir -p ./target/artifacts
|
||||
cp target/debug/rustfs ./target/artifacts/rustfs-debug
|
||||
# cp target/release/rustfs ./target/artifacts/rustfs-release
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: rustfs
|
||||
path: ./target/artifacts/*
|
||||
|
||||
test:
|
||||
name: Test
|
||||
needs:
|
||||
- skip-check
|
||||
- develop
|
||||
if: needs.skip-check.outputs.should_skip != 'true'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup
|
||||
with:
|
||||
cache-shared-key: "develop"
|
||||
cache-save-if: "false"
|
||||
- run: cargo test --all --exclude e2e_test
|
||||
|
||||
s3s-e2e:
|
||||
name: E2E (s3s-e2e)
|
||||
needs:
|
||||
@@ -96,10 +72,11 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: ./.github/actions/setup
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-shared-key: "develop"
|
||||
cache-save-if: false
|
||||
cache-on-failure: true
|
||||
cache-all-crates: true
|
||||
|
||||
- name: Install s3s-e2e
|
||||
run: |
|
||||
@@ -111,65 +88,12 @@ jobs:
|
||||
name: rustfs
|
||||
path: ./target/artifacts
|
||||
|
||||
- name: Run rustfs
|
||||
run: |
|
||||
./scripts/e2e-run.sh ./target/artifacts/rustfs-debug /data/rustfs
|
||||
sleep 10
|
||||
|
||||
- name: Run s3s-e2e
|
||||
timeout-minutes: 10
|
||||
run: |
|
||||
export AWS_ACCESS_KEY_ID=rustfsadmin
|
||||
export AWS_SECRET_ACCESS_KEY=rustfsadmin
|
||||
export AWS_REGION=us-east-1
|
||||
export AWS_ENDPOINT_URL=http://localhost:9000
|
||||
export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug"
|
||||
export RUST_BACKTRACE=full
|
||||
s3s-e2e
|
||||
sudo killall rustfs-debug
|
||||
./scripts/e2e-run.sh ./target/artifacts/rustfs-debug /tmp/rustfs
|
||||
|
||||
- uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: s3s-e2e.logs
|
||||
path: /tmp/rustfs.log
|
||||
|
||||
# mint:
|
||||
# name: E2E (mint)
|
||||
# needs:
|
||||
# - skip-check
|
||||
# - develop
|
||||
# if: needs.skip-check.outputs.should_skip != 'true'
|
||||
# runs-on: ubuntu-latest
|
||||
# steps:
|
||||
# - uses: actions/checkout@v4
|
||||
# - uses: ./.github/actions/setup
|
||||
# with:
|
||||
# cache-shared-key: "develop"
|
||||
# cache-save-if: false
|
||||
|
||||
# - uses: actions/download-artifact@v4
|
||||
# with:
|
||||
# name: rustfs
|
||||
# path: ./target/artifacts
|
||||
|
||||
# - name: Run rustfs
|
||||
# run: |
|
||||
# ./scripts/e2e-run.sh ./target/artifacts/rustfs-release /data/rustfs
|
||||
# sleep 10
|
||||
|
||||
# - name: Run mint
|
||||
# timeout-minutes: 10
|
||||
# run: |
|
||||
# docker run \
|
||||
# -e "SERVER_ENDPOINT=localhost:9000" \
|
||||
# -e "ACCESS_KEY=rustfsadmin" \
|
||||
# -e "SECRET_KEY=rustfsadmin" \
|
||||
# -e "ENABLE_HTTPS=0" \
|
||||
# --network host \
|
||||
# minio/mint:edge
|
||||
# sudo killall rustfs-release
|
||||
|
||||
# - uses: actions/upload-artifact@v4
|
||||
# with:
|
||||
# name: mint.logs
|
||||
# path: /tmp/rustfs.log
|
||||
|
||||
2
.gitignore
vendored
2
.gitignore
vendored
@@ -6,3 +6,5 @@
|
||||
/logs
|
||||
.devcontainer
|
||||
rustfs/static/*
|
||||
vendor
|
||||
cli/rustfs-gui/embedded-rustfs/rustfs
|
||||
563
Cargo.lock
generated
563
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
45
Cargo.toml
45
Cargo.toml
@@ -1,19 +1,19 @@
|
||||
[workspace]
|
||||
members = [
|
||||
"madmin",
|
||||
"rustfs",
|
||||
"ecstore",
|
||||
"e2e_test",
|
||||
"common/common",
|
||||
"common/lock",
|
||||
"common/protos",
|
||||
"api/admin",
|
||||
"reader",
|
||||
"common/workers",
|
||||
"iam",
|
||||
"crypto",
|
||||
"cli/rustfs-gui",
|
||||
"packages/logging",
|
||||
"madmin", # Management dashboard and admin API interface
|
||||
"rustfs", # Core file system implementation
|
||||
"ecstore", # Erasure coding storage implementation
|
||||
"e2e_test", # End-to-end test suite
|
||||
"common/common", # Shared utilities and data structures
|
||||
"common/lock", # Distributed locking implementation
|
||||
"common/protos", # Protocol buffer definitions
|
||||
"api/admin", # Admin HTTP API endpoints
|
||||
"reader", # Object reading service
|
||||
"common/workers", # Worker thread pools and task scheduling
|
||||
"iam", # Identity and Access Management
|
||||
"crypto", # Cryptography and security features
|
||||
"cli/rustfs-gui", # Graphical user interface client
|
||||
"packages/logging", # Logging utilities
|
||||
]
|
||||
resolver = "2"
|
||||
|
||||
@@ -36,8 +36,8 @@ async-trait = "0.1.86"
|
||||
backon = "1.3.0"
|
||||
bytes = "1.9.0"
|
||||
bytesize = "1.3.0"
|
||||
chrono = { version = "0.4.39", features = ["serde"] }
|
||||
clap = { version = "4.5.27", features = ["derive", "env"] }
|
||||
chrono = { version = "0.4.40", features = ["serde"] }
|
||||
clap = { version = "4.5.31", features = ["derive", "env"] }
|
||||
dioxus = { version = "0.6.3", features = ["router"] }
|
||||
dirs = "6.0.0"
|
||||
ecstore = { path = "./ecstore" }
|
||||
@@ -46,6 +46,7 @@ futures = "0.3.31"
|
||||
futures-util = "0.3.31"
|
||||
common = { path = "./common/common" }
|
||||
reader = { path = "./reader" }
|
||||
hex = "0.4.3"
|
||||
hyper = "1.6.0"
|
||||
hyper-util = { version = "0.1.10", features = [
|
||||
"tokio",
|
||||
@@ -58,6 +59,7 @@ humantime = "2.1.0"
|
||||
keyring = { version = "3.6.1", features = ["apple-native", "windows-native", "sync-secret-service"] }
|
||||
lock = { path = "./common/lock" }
|
||||
lazy_static = "1.5.0"
|
||||
local-ip-address = "0.6.3"
|
||||
mime = "0.3.17"
|
||||
netif = "0.1.6"
|
||||
opentelemetry = { version = "0.28" }
|
||||
@@ -80,6 +82,7 @@ rfd = { version = "0.15.2", default-features = false, features = ["xdg-portal",
|
||||
rmp = "0.8.14"
|
||||
rmp-serde = "1.3.0"
|
||||
rustfs-logging = { path = "packages/logging", version = "0.0.1" }
|
||||
rust-embed = "8.6.0"
|
||||
s3s = { git = "https://github.com/Nugine/s3s.git", rev = "ab139f72fe768fb9d8cecfe36269451da1ca9779", default-features = true, features = [
|
||||
"tower",
|
||||
] }
|
||||
@@ -87,6 +90,7 @@ s3s-policy = { git = "https://github.com/Nugine/s3s.git", rev = "ab139f72fe768fb
|
||||
shadow-rs = { version = "0.38.0", default-features = false }
|
||||
serde = { version = "1.0.217", features = ["derive"] }
|
||||
serde_json = "1.0.138"
|
||||
sha2 = "0.10.8"
|
||||
tempfile = "3.16.0"
|
||||
thiserror = "2.0.11"
|
||||
time = { version = "0.3.37", features = [
|
||||
@@ -109,7 +113,7 @@ tracing-appender = "0.2.3"
|
||||
tracing-opentelemetry = "0.29"
|
||||
transform-stream = "0.3.1"
|
||||
url = "2.5.4"
|
||||
uuid = { version = "1.12.1", features = [
|
||||
uuid = { version = "1.15.1", features = [
|
||||
"v4",
|
||||
"fast-rng",
|
||||
"macro-diagnostics",
|
||||
@@ -119,10 +123,7 @@ axum = "0.7.9"
|
||||
md-5 = "0.10.6"
|
||||
workers = { path = "./common/workers" }
|
||||
test-case = "3.3.1"
|
||||
zip = "2.2.2"
|
||||
|
||||
|
||||
[profile]
|
||||
zip = "2.2.3"
|
||||
|
||||
[profile.wasm-dev]
|
||||
inherits = "dev"
|
||||
@@ -137,4 +138,4 @@ inherits = "dev"
|
||||
[profile.release]
|
||||
opt-level = 3 # Optimization Level (0-3)
|
||||
lto = true # Optimize when linking
|
||||
codegen-units = 1 # Reduce code generation units to improve optimization
|
||||
codegen-units = 1 # Reduce code generation units to improve optimization
|
||||
|
||||
51
README.md
51
README.md
@@ -1 +1,50 @@
|
||||
# s3-rustfs
|
||||
# How to compile RustFS
|
||||
|
||||
| Must package | Version |
|
||||
|--------------|---------|
|
||||
| Rust | 1.8.5 |
|
||||
| protoc | 27.0 |
|
||||
| flatc | 24.0+ |
|
||||
|
||||
Download Links:
|
||||
|
||||
https://github.com/google/flatbuffers/releases/download/v24.3.25/Linux.flatc.binary.g++-13.zip
|
||||
|
||||
https://github.com/protocolbuffers/protobuf/releases/download/v27.0/protoc-27.0-linux-x86_64.zip
|
||||
|
||||
Or use Docker:
|
||||
|
||||
- uses: arduino/setup-protoc@v3
|
||||
with:
|
||||
version: "27.0"
|
||||
|
||||
- uses: Nugine/setup-flatc@v1
|
||||
with:
|
||||
version: "24.3.25"
|
||||
|
||||
# How to add Console web
|
||||
|
||||
1. wget [http://dl.rustfs.com/console/console.latest.tar.gz](https://dl.rustfs.com/console/rustfs-console-latest.zip)
|
||||
|
||||
2. mkdir in this repos folder `./rustfs/static`
|
||||
|
||||
3. Compile RustFS
|
||||
|
||||
# Star RustFS
|
||||
|
||||
Add Env infomation:
|
||||
|
||||
```
|
||||
export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug"
|
||||
export RUSTFS_VOLUMES="./target/volume/test"
|
||||
export RUSTFS_ADDRESS="0.0.0.0:9000"
|
||||
export RUSTFS_CONSOLE_ENABLE=true
|
||||
export RUSTFS_CONSOLE_ADDRESS="0.0.0.0:9001"
|
||||
export RUSTFS_SERVER_ENDPOINT="http://127.0.0.1:9000"
|
||||
```
|
||||
|
||||
You need replace your real data folder:
|
||||
|
||||
```
|
||||
./rustfs /data/rustfs
|
||||
```
|
||||
|
||||
@@ -11,17 +11,17 @@ chrono = { workspace = true }
|
||||
dioxus = { workspace = true, features = ["router"] }
|
||||
dirs = { workspace = true }
|
||||
futures-util = { workspace = true }
|
||||
hex = { workspace = true }
|
||||
keyring = { workspace = true }
|
||||
reqwest = { workspace = true }
|
||||
lazy_static = { workspace = true }
|
||||
rfd = { workspace = true }
|
||||
rust-embed = { workspace = true, features = ["interpolate-folder-path"] }
|
||||
serde = { workspace = true }
|
||||
serde_json = { workspace = true }
|
||||
sha2 = { workspace = true }
|
||||
tokio = { workspace = true, features = ["io-util", "net", "process", "sync"] }
|
||||
tracing-subscriber = { workspace = true, features = ["fmt", "env-filter", "tracing-log", "time", "local-time", "json"] }
|
||||
tracing-appender = { workspace = true }
|
||||
zip = { workspace = true }
|
||||
|
||||
|
||||
|
||||
[features]
|
||||
default = ["desktop"]
|
||||
@@ -29,22 +29,5 @@ web = ["dioxus/web"]
|
||||
desktop = ["dioxus/desktop"]
|
||||
mobile = ["dioxus/mobile"]
|
||||
|
||||
[profile]
|
||||
|
||||
[profile.wasm-dev]
|
||||
inherits = "dev"
|
||||
opt-level = 1
|
||||
|
||||
[profile.server-dev]
|
||||
inherits = "dev"
|
||||
|
||||
[profile.android-dev]
|
||||
inherits = "dev"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3 # Optimization Level (0-3)
|
||||
lto = true # Optimize when linking
|
||||
codegen-units = 1 # Reduce code generation units to improve optimization
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
1
cli/rustfs-gui/embedded-rustfs/README.md
Normal file
1
cli/rustfs-gui/embedded-rustfs/README.md
Normal file
@@ -0,0 +1 @@
|
||||
rustfs bin path, do not delete
|
||||
@@ -1,5 +1,5 @@
|
||||
use crate::components::navbar::LoadingSpinner;
|
||||
use crate::router::Route;
|
||||
use crate::route::Route;
|
||||
use crate::utils::{RustFSConfig, ServiceManager};
|
||||
use chrono::Datelike;
|
||||
use dioxus::logger::tracing::debug;
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::router::Route;
|
||||
use crate::route::Route;
|
||||
use dioxus::logger::tracing::debug;
|
||||
use dioxus::prelude::*;
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
mod components;
|
||||
mod router;
|
||||
mod route;
|
||||
mod utils;
|
||||
mod views;
|
||||
|
||||
|
||||
@@ -1,14 +1,31 @@
|
||||
use crate::utils::RustFSConfig;
|
||||
use dioxus::logger::tracing::{debug, error, info};
|
||||
use futures_util::TryStreamExt;
|
||||
use lazy_static::lazy_static;
|
||||
use rust_embed::RustEmbed;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::error::Error;
|
||||
use std::fs::File;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::Command as StdCommand;
|
||||
use std::time::Duration;
|
||||
use tokio::fs;
|
||||
use tokio::fs::File;
|
||||
use tokio::io::AsyncWriteExt;
|
||||
use tokio::net::TcpStream;
|
||||
use tokio::sync::mpsc;
|
||||
use tokio::sync::{mpsc, Mutex};
|
||||
|
||||
#[derive(RustEmbed)]
|
||||
#[folder = "$CARGO_MANIFEST_DIR/embedded-rustfs/"]
|
||||
struct Asset;
|
||||
|
||||
// Use `lazy_static` to cache the checksum of embedded resources
|
||||
lazy_static! {
|
||||
static ref RUSTFS_HASH: Mutex<String> = {
|
||||
let rustfs_file = if cfg!(windows) { "rustfs.exe" } else { "rustfs" };
|
||||
let rustfs_data = Asset::get(rustfs_file).expect("RustFs binary not embedded");
|
||||
let hash = hex::encode(Sha256::digest(&rustfs_data.data));
|
||||
Mutex::new(hash)
|
||||
};
|
||||
}
|
||||
|
||||
/// Service command
|
||||
/// This enum represents the commands that can be sent to the service manager
|
||||
@@ -159,107 +176,38 @@ impl ServiceManager {
|
||||
}
|
||||
}
|
||||
|
||||
let executable_path = if cfg!(windows) {
|
||||
bin_dir.join("rustfs.exe")
|
||||
} else {
|
||||
bin_dir.join("rustfs")
|
||||
};
|
||||
let rustfs_file = if cfg!(windows) { "rustfs.exe" } else { "rustfs" };
|
||||
let executable_path = bin_dir.join(rustfs_file);
|
||||
let hash_path = bin_dir.join("embedded_rustfs.sha256");
|
||||
|
||||
// If the executable file doesn't exist, download and unzip it
|
||||
if !executable_path.exists() {
|
||||
// download the file
|
||||
let tmp_zip = rustfs_dir.join("rustfs.zip");
|
||||
let file_download_url = if cfg!(windows) {
|
||||
"https://api.xmb.xyz/download/rustfs-win.zip"
|
||||
} else {
|
||||
"https://api.xmb.xyz/download/rustfs.zip"
|
||||
};
|
||||
|
||||
let download_task = Self::download_file(file_download_url, &tmp_zip);
|
||||
let unzip_task = async {
|
||||
download_task.await?;
|
||||
Self::unzip_file(&tmp_zip, &bin_dir)?;
|
||||
tokio::fs::remove_file(&tmp_zip).await?;
|
||||
Ok::<(), Box<dyn Error>>(())
|
||||
};
|
||||
unzip_task.await?;
|
||||
|
||||
// delete the temporary zip file
|
||||
tokio::fs::remove_file(&tmp_zip).await?;
|
||||
|
||||
// set execution permissions on unix systems
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mut perms = std::fs::metadata(&executable_path)?.permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&executable_path, perms)?;
|
||||
if executable_path.exists() && hash_path.exists() {
|
||||
let cached_hash = fs::read_to_string(&hash_path).await?;
|
||||
let expected_hash = RUSTFS_HASH.lock().await;
|
||||
if cached_hash == *expected_hash {
|
||||
println!("Use cached rustfs: {:?}", executable_path);
|
||||
return Ok(executable_path);
|
||||
}
|
||||
Self::show_info("服务程序已成功下载并准备就绪");
|
||||
}
|
||||
|
||||
// Extract and write files
|
||||
let rustfs_data = Asset::get(rustfs_file).expect("RustFS binary not embedded");
|
||||
let mut file = File::create(&executable_path).await?;
|
||||
file.write_all(&rustfs_data.data).await?;
|
||||
let expected_hash = hex::encode(Sha256::digest(&rustfs_data.data));
|
||||
fs::write(&hash_path, expected_hash).await?;
|
||||
|
||||
// set execution permissions on unix systems
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mut perms = std::fs::metadata(&executable_path)?.permissions();
|
||||
perms.set_mode(0o755);
|
||||
std::fs::set_permissions(&executable_path, perms)?;
|
||||
}
|
||||
|
||||
Ok(executable_path)
|
||||
}
|
||||
|
||||
/// Download the file
|
||||
/// This function is a modified version of the example from the `reqwest` crate documentation
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// let url = "https://api.xmb.xyz/download/rustfs.zip";
|
||||
/// let path = Path::new("rustfs.zip");
|
||||
/// download_file(url, path).await;
|
||||
/// ```
|
||||
async fn download_file(url: &str, path: &Path) -> Result<(), Box<dyn Error>> {
|
||||
let client = reqwest::Client::new();
|
||||
let res = client.get(url).send().await?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
return Err(format!("下载失败:{}", res.status()).into());
|
||||
}
|
||||
|
||||
let mut file = tokio::fs::File::create(path).await?;
|
||||
let mut stream = res.bytes_stream();
|
||||
while let Ok(Some(chunk)) = stream.try_next().await {
|
||||
file.write_all(&chunk).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// unzip the file
|
||||
/// This function is a modified version of the example from the `zip` crate documentation
|
||||
///
|
||||
/// # Example
|
||||
/// ```
|
||||
/// let zip_path = Path::new("rustfs.zip");
|
||||
/// let extract_path = Path::new("rustfs");
|
||||
/// unzip_file(zip_path, extract_path);
|
||||
/// ```
|
||||
fn unzip_file(zip_path: &Path, extract_path: &Path) -> Result<(), Box<dyn Error>> {
|
||||
let file = File::open(zip_path)?;
|
||||
let mut archive = zip::ZipArchive::new(file)?;
|
||||
|
||||
for i in 0..archive.len() {
|
||||
let mut file = archive.by_index(i)?;
|
||||
let out_path = extract_path.join(file.name());
|
||||
|
||||
if file.name().ends_with('/') {
|
||||
std::fs::create_dir_all(&out_path)?;
|
||||
} else {
|
||||
if let Some(p) = out_path.parent() {
|
||||
if !p.exists() {
|
||||
std::fs::create_dir_all(p)?;
|
||||
}
|
||||
}
|
||||
let mut outfile = File::create(&out_path)?;
|
||||
std::io::copy(&mut file, &mut outfile)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Helper function: Extracts the port from the address string
|
||||
///
|
||||
/// # Example
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::router::Route;
|
||||
use crate::route::Route;
|
||||
use dioxus::logger::tracing::info;
|
||||
use dioxus::prelude::*;
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ s3s.workspace = true
|
||||
http.workspace = true
|
||||
highway = "1.3.0"
|
||||
url.workspace = true
|
||||
uuid = { version = "1.12.1", features = ["v4", "fast-rng", "serde"] }
|
||||
uuid = { version = "1.15.1", features = ["v4", "fast-rng", "serde"] }
|
||||
reed-solomon-erasure = { version = "6.0.0", features = ["simd-accel"] }
|
||||
transform-stream = "0.3.1"
|
||||
lazy_static.workspace = true
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
use crate::error::{Error, Result};
|
||||
use crate::utils::net;
|
||||
use path_absolutize::Absolutize;
|
||||
use path_clean::PathClean;
|
||||
use std::{fmt::Display, path::Path};
|
||||
use url::{ParseError, Url};
|
||||
|
||||
@@ -29,7 +28,7 @@ pub struct Endpoint {
|
||||
impl Display for Endpoint {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
if self.url.scheme() == "file" {
|
||||
write!(f, "{}", self.url.path())
|
||||
write!(f, "{}", self.get_file_path())
|
||||
} else {
|
||||
write!(f, "{}", self.url)
|
||||
}
|
||||
@@ -42,12 +41,8 @@ impl TryFrom<&str> for Endpoint {
|
||||
|
||||
/// Performs the conversion.
|
||||
fn try_from(value: &str) -> Result<Self, Self::Error> {
|
||||
/// check whether given path is not empty.
|
||||
fn is_empty_path(path: impl AsRef<Path>) -> bool {
|
||||
["", "/", "\\"].iter().any(|&v| Path::new(v).eq(path.as_ref()))
|
||||
}
|
||||
|
||||
if is_empty_path(value) {
|
||||
// check whether given path is not empty.
|
||||
if ["", "/", "\\"].iter().any(|&v| v.eq(value)) {
|
||||
return Err(Error::from_string("empty or root endpoint is not supported"));
|
||||
}
|
||||
|
||||
@@ -67,35 +62,26 @@ impl TryFrom<&str> for Endpoint {
|
||||
return Err(Error::from_string("invalid URL endpoint format"));
|
||||
}
|
||||
|
||||
let path = Path::new(url.path()).clean();
|
||||
if is_empty_path(&path) {
|
||||
return Err(Error::from_string("empty or root path is not supported in URL endpoint"));
|
||||
}
|
||||
let path = url.path().to_string();
|
||||
|
||||
if let Some(v) = path.to_str() {
|
||||
url.set_path(v)
|
||||
}
|
||||
#[cfg(not(windows))]
|
||||
let path = Path::new(&path).absolutize()?;
|
||||
|
||||
// On windows having a preceding SlashSeparator will cause problems, if the
|
||||
// command line already has C:/<export-folder/ in it. Final resulting
|
||||
// path on windows might become C:/C:/ this will cause problems
|
||||
// of starting rustfs server properly in distributed mode on windows.
|
||||
// As a special case make sure to trim the separator.
|
||||
|
||||
// NOTE: It is also perfectly fine for windows users to have a path
|
||||
// without C:/ since at that point we treat it as relative path
|
||||
// and obtain the full filesystem path as well. Providing C:/
|
||||
// style is necessary to provide paths other than C:/,
|
||||
// such as F:/, D:/ etc.
|
||||
//
|
||||
// Another additional benefit here is that this style also
|
||||
// supports providing \\host\share support as well.
|
||||
#[cfg(windows)]
|
||||
{
|
||||
let path = url.path().to_owned();
|
||||
if Path::new(&path[1..]).is_absolute() {
|
||||
url.set_path(&path[1..]);
|
||||
}
|
||||
let path = Path::new(&path[1..]).absolutize()?;
|
||||
|
||||
if path.parent().is_none() || Path::new("").eq(&path) {
|
||||
return Err(Error::from_string("empty or root path is not supported in URL endpoint"));
|
||||
}
|
||||
|
||||
match path.to_str() {
|
||||
Some(v) => url.set_path(v),
|
||||
None => return Err(Error::from_string("invalid path")),
|
||||
}
|
||||
|
||||
url
|
||||
@@ -182,6 +168,15 @@ impl Endpoint {
|
||||
_ => String::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn get_file_path(&self) -> &str {
|
||||
let path = self.url.path();
|
||||
|
||||
#[cfg(windows)]
|
||||
let path = &path[1..];
|
||||
|
||||
path
|
||||
}
|
||||
}
|
||||
|
||||
/// parse a file path into an URL.
|
||||
|
||||
@@ -114,7 +114,7 @@ impl Debug for LocalDisk {
|
||||
|
||||
impl LocalDisk {
|
||||
pub async fn new(ep: &Endpoint, cleanup: bool) -> Result<Self> {
|
||||
let root = fs::canonicalize(ep.url.path()).await?;
|
||||
let root = fs::canonicalize(ep.get_file_path()).await?;
|
||||
|
||||
if cleanup {
|
||||
// TODO: 删除tmp数据
|
||||
|
||||
@@ -52,7 +52,7 @@ pub struct RemoteDisk {
|
||||
impl RemoteDisk {
|
||||
pub async fn new(ep: &Endpoint, _opt: &DiskOption) -> Result<Self> {
|
||||
// let root = fs::canonicalize(ep.url.path()).await?;
|
||||
let root = PathBuf::from(ep.url.path());
|
||||
let root = PathBuf::from(ep.get_file_path());
|
||||
let addr = format!("{}://{}:{}", ep.url.scheme(), ep.url.host_str().unwrap(), ep.url.port().unwrap());
|
||||
Ok(Self {
|
||||
id: Mutex::new(None),
|
||||
|
||||
@@ -231,7 +231,7 @@ impl PoolEndpointList {
|
||||
.map_err(|e| Error::from_string(format!("host '{}' cannot resolve: {}", host, e)))?
|
||||
});
|
||||
|
||||
let path = ep.url.path();
|
||||
let path = ep.get_file_path();
|
||||
match path_ip_map.entry(path) {
|
||||
Entry::Occupied(mut e) => {
|
||||
if e.get().intersection(host_ip_set).count() > 0 {
|
||||
@@ -255,7 +255,7 @@ impl PoolEndpointList {
|
||||
continue;
|
||||
}
|
||||
|
||||
let path = ep.url.path();
|
||||
let path = ep.get_file_path();
|
||||
if local_path_set.contains(path) {
|
||||
return Err(Error::from_string(format!(
|
||||
"path '{}' cannot be served by different address on same server",
|
||||
@@ -272,7 +272,7 @@ impl PoolEndpointList {
|
||||
let mut local_endpoint_count = 0;
|
||||
|
||||
for ep in endpoints.as_ref() {
|
||||
ep_path_set.insert(ep.url.path());
|
||||
ep_path_set.insert(ep.get_file_path());
|
||||
if ep.is_local && ep.url.has_host() {
|
||||
local_server_host_set.insert(ep.url.host());
|
||||
local_port_set.insert(ep.url.port());
|
||||
|
||||
@@ -18,7 +18,7 @@ ecstore = { path = "../ecstore" }
|
||||
serde_json.workspace = true
|
||||
async-trait.workspace = true
|
||||
thiserror.workspace = true
|
||||
strum = { version = "0.26.3", features = ["derive"] }
|
||||
strum = { version = "0.27.1", features = ["derive"] }
|
||||
arc-swap = "1.7.1"
|
||||
crypto = { path = "../crypto" }
|
||||
ipnetwork = { version = "0.21.1", features = ["serde"] }
|
||||
|
||||
@@ -152,10 +152,7 @@ impl Credentials {
|
||||
const IAM_POLICY_CLAIM_NAME_SA: &str = "sa-policy";
|
||||
self.claims
|
||||
.as_ref()
|
||||
.map(|x| {
|
||||
x.get(IAM_POLICY_CLAIM_NAME_SA)
|
||||
.map_or(false, |_| !self.parent_user.is_empty())
|
||||
})
|
||||
.map(|x| x.get(IAM_POLICY_CLAIM_NAME_SA).is_some_and(|_| !self.parent_user.is_empty()))
|
||||
.unwrap_or_default()
|
||||
}
|
||||
|
||||
@@ -164,10 +161,7 @@ impl Credentials {
|
||||
return self
|
||||
.claims
|
||||
.as_ref()
|
||||
.map(|x| {
|
||||
x.get(&iam_policy_claim_name_sa())
|
||||
.map_or(false, |v| v == INHERITED_POLICY_TYPE)
|
||||
})
|
||||
.map(|x| x.get(&iam_policy_claim_name_sa()).is_some_and(|v| v == INHERITED_POLICY_TYPE))
|
||||
.unwrap_or_default();
|
||||
}
|
||||
|
||||
|
||||
@@ -79,6 +79,9 @@ pub enum Error {
|
||||
#[error("action not allowed")]
|
||||
IAMActionNotAllowed,
|
||||
|
||||
#[error("invalid expiration")]
|
||||
InvalidExpiration,
|
||||
|
||||
#[error("no secret key with access key")]
|
||||
NoSecretKeyWithAccessKey,
|
||||
|
||||
|
||||
@@ -131,9 +131,9 @@ impl<'de> Deserialize<'de> for Functions {
|
||||
}
|
||||
}
|
||||
|
||||
if inner_data.is_empty() {
|
||||
/* if inner_data.is_empty() {
|
||||
return Err(Error::custom("has no condition element"));
|
||||
}
|
||||
} */
|
||||
|
||||
Ok(inner_data)
|
||||
}
|
||||
@@ -180,7 +180,7 @@ mod tests {
|
||||
"s3:x-amz-server-side-encryption-customer-algorithm": "true"
|
||||
}
|
||||
}"# => false; "1")]
|
||||
#[test_case(r#"{}"# => false; "2")]
|
||||
#[test_case(r#"{}"# => true; "2")]
|
||||
#[test_case(
|
||||
r#"{
|
||||
"StringLike": {
|
||||
|
||||
@@ -846,7 +846,9 @@ impl Store for ObjectStore {
|
||||
let name = ecstore::utils::path::dir(item);
|
||||
info!("load svc user: {}", name);
|
||||
if let Err(err) = self.load_user(&name, UserType::Svc, &mut items_cache).await {
|
||||
return Err(Error::msg(std::format!("load_user failed: {}", err)));
|
||||
if !is_err_no_such_user(&err) {
|
||||
return Err(Error::msg(std::format!("load svc user failed: {}", err)));
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -197,6 +197,10 @@ impl<T: Store> IamSys<T> {
|
||||
return Err(IamError::IAMActionNotAllowed.into());
|
||||
}
|
||||
|
||||
if opts.expiration.is_none() {
|
||||
return Err(IamError::InvalidExpiration.into());
|
||||
}
|
||||
|
||||
// TODO: check allow_site_replicator_account
|
||||
|
||||
let policy_buf = if let Some(policy) = opts.session_policy {
|
||||
@@ -229,9 +233,13 @@ impl<T: Store> IamSys<T> {
|
||||
}
|
||||
}
|
||||
|
||||
// set expiration time default to 1 hour
|
||||
m.insert(
|
||||
"exp".to_string(),
|
||||
serde_json::Value::Number(serde_json::Number::from(opts.expiration.map_or(0, |t| t.unix_timestamp()))),
|
||||
serde_json::Value::Number(serde_json::Number::from(
|
||||
opts.expiration
|
||||
.map_or(OffsetDateTime::now_utc().unix_timestamp() + 3600, |t| t.unix_timestamp()),
|
||||
)),
|
||||
);
|
||||
|
||||
let (access_key, secret_key) = if !opts.access_key.is_empty() || !opts.secret_key.is_empty() {
|
||||
@@ -385,7 +393,7 @@ impl<T: Store> IamSys<T> {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
if u.credentials.is_service_account() {
|
||||
if !u.credentials.is_service_account() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use ecstore::error::{Error, Result};
|
||||
use jsonwebtoken::{encode, Algorithm, DecodingKey, EncodingKey, Header};
|
||||
use jsonwebtoken::{Algorithm, DecodingKey, EncodingKey, Header};
|
||||
use rand::{Rng, RngCore};
|
||||
use serde::{de::DeserializeOwned, Serialize};
|
||||
|
||||
@@ -42,7 +42,7 @@ pub fn gen_secret_key(length: usize) -> crate::Result<String> {
|
||||
|
||||
pub fn generate_jwt<T: Serialize>(claims: &T, secret: &str) -> Result<String, jsonwebtoken::errors::Error> {
|
||||
let header = Header::new(Algorithm::HS512);
|
||||
encode(&header, &claims, &EncodingKey::from_secret(secret.as_bytes()))
|
||||
jsonwebtoken::encode(&header, &claims, &EncodingKey::from_secret(secret.as_bytes()))
|
||||
}
|
||||
|
||||
pub fn extract_claims<T: DeserializeOwned>(
|
||||
@@ -58,7 +58,8 @@ pub fn extract_claims<T: DeserializeOwned>(
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{gen_access_key, gen_secret_key};
|
||||
use super::{gen_access_key, gen_secret_key, generate_jwt};
|
||||
use serde::{Deserialize, Serialize};
|
||||
|
||||
#[test]
|
||||
fn test_gen_access_key() {
|
||||
@@ -76,4 +77,34 @@ mod tests {
|
||||
let b = gen_secret_key(10).unwrap();
|
||||
assert_ne!(a, b);
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize, Deserialize, PartialEq)]
|
||||
struct Claims {
|
||||
sub: String,
|
||||
company: String,
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_generate_jwt() {
|
||||
let claims = Claims {
|
||||
sub: "user1".to_string(),
|
||||
company: "example".to_string(),
|
||||
};
|
||||
let secret = "my_secret";
|
||||
let token = generate_jwt(&claims, secret).unwrap();
|
||||
|
||||
assert!(!token.is_empty());
|
||||
}
|
||||
|
||||
// #[test]
|
||||
// fn test_extract_claims() {
|
||||
// let claims = Claims {
|
||||
// sub: "user1".to_string(),
|
||||
// company: "example".to_string(),
|
||||
// };
|
||||
// let secret = "my_secret";
|
||||
// let token = generate_jwt(&claims, secret).unwrap();
|
||||
// let decoded_claims = extract_claims::<Claims>(&token, secret).unwrap();
|
||||
// assert_eq!(decoded_claims.claims, claims);
|
||||
// }
|
||||
}
|
||||
|
||||
@@ -211,6 +211,7 @@ pub struct UpdateServiceAccountReq {
|
||||
pub new_description: Option<String>,
|
||||
|
||||
#[serde(rename = "newExpiration", skip_serializing_if = "Option::is_none")]
|
||||
#[serde(with = "time::serde::rfc3339::option")]
|
||||
pub new_expiration: Option<OffsetDateTime>,
|
||||
}
|
||||
|
||||
|
||||
@@ -59,7 +59,7 @@ tower.workspace = true
|
||||
tracing-error.workspace = true
|
||||
tracing-subscriber.workspace = true
|
||||
transform-stream.workspace = true
|
||||
uuid = "1.12.1"
|
||||
uuid = "1.15.1"
|
||||
url.workspace = true
|
||||
admin = { path = "../api/admin" }
|
||||
axum.workspace = true
|
||||
@@ -73,7 +73,9 @@ iam = { path = "../iam" }
|
||||
jsonwebtoken = "9.3.0"
|
||||
tower-http = { version = "0.6.2", features = ["cors"] }
|
||||
mime_guess = "2.0.5"
|
||||
rust-embed = { version = "8.5.0", features = ["interpolate-folder-path"] }
|
||||
rust-embed = { workspace = true, features = ["interpolate-folder-path"] }
|
||||
local-ip-address = { workspace = true }
|
||||
chrono = { workspace = true }
|
||||
|
||||
[build-dependencies]
|
||||
prost-build.workspace = true
|
||||
@@ -85,7 +87,7 @@ futures-util.workspace = true
|
||||
# uuid = { version = "1.8.0", features = ["v4", "fast-rng", "serde"] }
|
||||
ecstore = { path = "../ecstore" }
|
||||
s3s.workspace = true
|
||||
clap = { version = "4.5.27", features = ["derive","env"] }
|
||||
clap = { version = "4.5.31", features = ["derive", "env"] }
|
||||
tracing-subscriber = { version = "0.3.19", features = ["env-filter", "time"] }
|
||||
hyper-util = { version = "0.1.10", features = [
|
||||
"tokio",
|
||||
|
||||
@@ -53,6 +53,16 @@ impl Operation for AddServiceAccount {
|
||||
.validate()
|
||||
.map_err(|e| S3Error::with_message(InvalidRequest, e.to_string()))?;
|
||||
|
||||
let session_policy = if let Some(policy) = &create_req.policy {
|
||||
let p = Policy::parse_config(policy.as_bytes()).map_err(|e| {
|
||||
debug!("parse policy failed, e: {:?}", e);
|
||||
s3_error!(InvalidArgument, "parse policy failed")
|
||||
})?;
|
||||
Some(p)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let Some(sys_cred) = get_global_action_cred() else {
|
||||
return Err(s3_error!(InvalidRequest, "get sys cred failed"));
|
||||
};
|
||||
@@ -95,7 +105,7 @@ impl Operation for AddServiceAccount {
|
||||
name: create_req.name,
|
||||
description: create_req.description,
|
||||
expiration: create_req.expiration,
|
||||
session_policy: create_req.policy.and_then(|p| Policy::parse_config(p.as_bytes()).ok()),
|
||||
session_policy,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
|
||||
@@ -60,6 +60,6 @@ pub struct Opt {
|
||||
#[arg(long, default_value_t = false, env = "RUSTFS_CONSOLE_ENABLE")]
|
||||
pub console_enable: bool,
|
||||
|
||||
#[arg(long, default_value_t = format!("127.0.0.1:{}", 0), env = "RUSTFS_CONSOLE_ADDRESS")]
|
||||
#[arg(long, default_value_t = format!("127.0.0.1:{}", 9002), env = "RUSTFS_CONSOLE_ADDRESS")]
|
||||
pub console_address: String,
|
||||
}
|
||||
|
||||
@@ -9,6 +9,12 @@ use axum::{
|
||||
use mime_guess::from_path;
|
||||
use rust_embed::RustEmbed;
|
||||
use serde::Serialize;
|
||||
use shadow_rs::shadow;
|
||||
use std::net::Ipv4Addr;
|
||||
use std::sync::OnceLock;
|
||||
use tracing::info;
|
||||
|
||||
shadow!(build);
|
||||
|
||||
#[derive(RustEmbed)]
|
||||
#[folder = "$CARGO_MANIFEST_DIR/static"]
|
||||
@@ -42,11 +48,12 @@ async fn static_handler(uri: axum::http::Uri) -> impl IntoResponse {
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
struct Config {
|
||||
pub(crate) struct Config {
|
||||
api: Api,
|
||||
s3: S3,
|
||||
release: Release,
|
||||
license: License,
|
||||
doc: String,
|
||||
}
|
||||
|
||||
impl Config {
|
||||
@@ -67,12 +74,30 @@ impl Config {
|
||||
name: "Apache-2.0".to_string(),
|
||||
url: "https://www.apache.org/licenses/LICENSE-2.0".to_string(),
|
||||
},
|
||||
doc: "https://rustfs.com/docs/".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn to_json(&self) -> String {
|
||||
serde_json::to_string(self).unwrap_or_default()
|
||||
}
|
||||
|
||||
pub(crate) fn version(&self) -> String {
|
||||
format!(
|
||||
"RELEASE.{} (rust {} {})",
|
||||
self.release.date.clone(),
|
||||
build::RUST_VERSION,
|
||||
build::BUILD_TARGET
|
||||
)
|
||||
}
|
||||
|
||||
pub(crate) fn license(&self) -> String {
|
||||
format!("{} {}", self.license.name.clone(), self.license.url.clone())
|
||||
}
|
||||
|
||||
pub(crate) fn doc(&self) -> String {
|
||||
self.doc.clone()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Serialize)]
|
||||
@@ -99,8 +124,27 @@ struct License {
|
||||
url: String,
|
||||
}
|
||||
|
||||
async fn config_handler(axum::extract::Extension(fs_addr): axum::extract::Extension<String>) -> impl IntoResponse {
|
||||
let cfg = Config::new(&fs_addr, "v0.0.1", "2025-01-01").to_json();
|
||||
pub(crate) static CONSOLE_CONFIG: OnceLock<Config> = OnceLock::new();
|
||||
|
||||
pub(crate) fn init_console_cfg(fs_addr: &str) {
|
||||
CONSOLE_CONFIG.get_or_init(|| {
|
||||
let ver = {
|
||||
if !build::TAG.is_empty() {
|
||||
build::TAG.to_string()
|
||||
} else if !build::SHORT_COMMIT.is_empty() {
|
||||
format!("@{}", build::SHORT_COMMIT)
|
||||
} else {
|
||||
build::PKG_VERSION.to_string()
|
||||
}
|
||||
};
|
||||
|
||||
Config::new(fs_addr, ver.as_str(), build::COMMIT_DATE_3339)
|
||||
});
|
||||
}
|
||||
|
||||
#[allow(clippy::const_is_empty)]
|
||||
async fn config_handler() -> impl IntoResponse {
|
||||
let cfg = CONSOLE_CONFIG.get().unwrap().to_json();
|
||||
|
||||
Response::builder()
|
||||
.header("content-type", "application/json")
|
||||
@@ -109,15 +153,18 @@ async fn config_handler(axum::extract::Extension(fs_addr): axum::extract::Extens
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
pub async fn start_static_file_server(addrs: &str, fs_addr: &str) {
|
||||
pub async fn start_static_file_server(addrs: &str, local_ip: Ipv4Addr, access_key: &str, secret_key: &str) {
|
||||
// 创建路由
|
||||
let app = Router::new()
|
||||
.route("/config.json", get(config_handler).layer(axum::extract::Extension(fs_addr.to_owned())))
|
||||
.route("/config.json", get(config_handler))
|
||||
.nest_service("/", get(static_handler));
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(addrs).await.unwrap();
|
||||
let local_addr = listener.local_addr().unwrap();
|
||||
|
||||
println!("console running on: http://{} with s3 api {}", listener.local_addr().unwrap(), fs_addr);
|
||||
info!("WebUI: http://{}:{} http://127.0.0.1:{}", local_ip, local_addr.port(), local_addr.port());
|
||||
info!(" RootUser: {}", access_key);
|
||||
info!(" RootPass: {}", secret_key);
|
||||
|
||||
axum::serve(listener, app).await.unwrap();
|
||||
}
|
||||
|
||||
@@ -6,13 +6,17 @@ mod grpc;
|
||||
mod logging;
|
||||
mod service;
|
||||
mod storage;
|
||||
mod utils;
|
||||
|
||||
use crate::auth::IAMAuth;
|
||||
use crate::console::{init_console_cfg, CONSOLE_CONFIG};
|
||||
use chrono::Datelike;
|
||||
use clap::Parser;
|
||||
use common::{
|
||||
error::{Error, Result},
|
||||
globals::set_global_addr,
|
||||
};
|
||||
use config::{DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY};
|
||||
use ecstore::heal::background_heal_ops::init_auto_heal;
|
||||
use ecstore::utils::net::{self, get_available_port};
|
||||
use ecstore::{
|
||||
@@ -39,18 +43,20 @@ use tonic::{metadata::MetadataValue, Request, Status};
|
||||
use tower_http::cors::CorsLayer;
|
||||
use tracing::{debug, error, info, warn};
|
||||
use tracing_error::ErrorLayer;
|
||||
use tracing_subscriber::{fmt, layer::SubscriberExt, util::SubscriberInitExt};
|
||||
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
|
||||
|
||||
fn setup_tracing() {
|
||||
use tracing_subscriber::EnvFilter;
|
||||
|
||||
let env_filter = EnvFilter::from_default_env();
|
||||
let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));
|
||||
let enable_color = std::io::stdout().is_terminal();
|
||||
|
||||
let subscriber = fmt()
|
||||
let subscriber = tracing_subscriber::fmt::fmt()
|
||||
.pretty()
|
||||
.with_env_filter(env_filter)
|
||||
.with_ansi(enable_color)
|
||||
.with_file(true)
|
||||
.with_line_number(true)
|
||||
.finish()
|
||||
.with(ErrorLayer::default());
|
||||
|
||||
@@ -66,6 +72,18 @@ fn check_auth(req: Request<()>) -> Result<Request<()>, Status> {
|
||||
}
|
||||
}
|
||||
|
||||
fn print_server_info() {
|
||||
let cfg = CONSOLE_CONFIG.get().unwrap();
|
||||
let current_year = chrono::Utc::now().year();
|
||||
|
||||
// 使用自定义宏打印服务器信息
|
||||
info!("RustFS Object Storage Server");
|
||||
info!("Copyright: 2024-{} RustFS, Inc", current_year);
|
||||
info!("License: {}", cfg.license());
|
||||
info!("Version: {}", cfg.version());
|
||||
info!("Docs: {}", cfg.doc());
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
//解析获得到的参数
|
||||
let opt = config::Opt::parse();
|
||||
@@ -102,13 +120,39 @@ async fn run(opt: config::Opt) -> Result<()> {
|
||||
let listener = TcpListener::bind(server_address.clone()).await?;
|
||||
//获取监听地址
|
||||
let local_addr: SocketAddr = listener.local_addr()?;
|
||||
let local_ip = utils::get_local_ip().ok_or(local_addr.ip()).unwrap();
|
||||
|
||||
// 用于 rpc
|
||||
let (endpoint_pools, setup_type) = EndpointServerPools::from_volumes(server_address.clone().as_str(), opt.volumes.clone())
|
||||
.map_err(|err| Error::from_string(err.to_string()))?;
|
||||
|
||||
// Print RustFS-style logging for pool formatting
|
||||
for (i, eps) in endpoint_pools.as_ref().iter().enumerate() {
|
||||
debug!(
|
||||
info!(
|
||||
"Formatting {}st pool, {} set(s), {} drives per set.",
|
||||
i + 1,
|
||||
eps.set_count,
|
||||
eps.drives_per_set
|
||||
);
|
||||
|
||||
// Add warning for host with multiple drives in a set (similar to RustFS)
|
||||
if eps.drives_per_set > 1 {
|
||||
warn!("WARNING: Host local has more than 0 drives of set. A host failure will result in data becoming unavailable.");
|
||||
}
|
||||
}
|
||||
|
||||
// Detailed endpoint information (showing all API endpoints)
|
||||
let api_endpoints = format!("http://{}:{}", local_ip, server_port);
|
||||
let localhost_endpoint = format!("http://127.0.0.1:{}", server_port);
|
||||
info!("API: {} {}", api_endpoints, localhost_endpoint);
|
||||
info!(" RootUser: {}", opt.access_key.clone());
|
||||
info!(" RootPass: {}", opt.secret_key.clone());
|
||||
if DEFAULT_ACCESS_KEY.eq(&opt.access_key) && DEFAULT_SECRET_KEY.eq(&opt.secret_key) {
|
||||
warn!("Detected default credentials '{}:{}', we recommend that you change these values with 'RUSTFS_ACCESS_KEY' and 'RUSTFS_SECRET_KEY' environment variables", DEFAULT_ACCESS_KEY, DEFAULT_SECRET_KEY);
|
||||
}
|
||||
|
||||
for (i, eps) in endpoint_pools.as_ref().iter().enumerate() {
|
||||
info!(
|
||||
"created endpoints {}, set_count:{}, drives_per_set: {}, cmd: {:?}",
|
||||
i, eps.set_count, eps.drives_per_set, eps.cmd_line
|
||||
);
|
||||
@@ -131,9 +175,12 @@ async fn run(opt: config::Opt) -> Result<()> {
|
||||
// let mut b = S3ServiceBuilder::new(storage::ecfs::FS::new(server_address.clone(), endpoint_pools).await?);
|
||||
let mut b = S3ServiceBuilder::new(store.clone());
|
||||
|
||||
let access_key = opt.access_key.clone();
|
||||
let secret_key = opt.secret_key.clone();
|
||||
//显示 info 信息
|
||||
info!("authentication is enabled {}, {}", &opt.access_key, &opt.secret_key);
|
||||
b.set_auth(IAMAuth::new(opt.access_key, opt.secret_key));
|
||||
debug!("authentication is enabled {}, {}", &access_key, &secret_key);
|
||||
|
||||
b.set_auth(IAMAuth::new(access_key, secret_key));
|
||||
|
||||
b.set_access(store.clone());
|
||||
|
||||
@@ -175,11 +222,10 @@ async fn run(opt: config::Opt) -> Result<()> {
|
||||
let http_server = ConnBuilder::new(TokioExecutor::new());
|
||||
let mut ctrl_c = std::pin::pin!(tokio::signal::ctrl_c());
|
||||
let graceful = hyper_util::server::graceful::GracefulShutdown::new();
|
||||
println!("server is running at http://{local_addr}");
|
||||
|
||||
loop {
|
||||
let (socket, _) = tokio::select! {
|
||||
res = listener.accept() => {
|
||||
res = listener.accept() => {
|
||||
match res {
|
||||
Ok(conn) => conn,
|
||||
Err(err) => {
|
||||
@@ -222,7 +268,7 @@ async fn run(opt: config::Opt) -> Result<()> {
|
||||
error!("ECStore init faild {:?}", &err);
|
||||
Error::from_string(err.to_string())
|
||||
})?;
|
||||
warn!(" init store success!");
|
||||
debug!("init store success!");
|
||||
|
||||
init_iam_sys(store.clone()).await.unwrap();
|
||||
|
||||
@@ -236,18 +282,17 @@ async fn run(opt: config::Opt) -> Result<()> {
|
||||
// init auto heal
|
||||
init_auto_heal().await;
|
||||
|
||||
info!("server was started");
|
||||
let srv_addr = format!("http://{}:{}", local_ip, server_port);
|
||||
init_console_cfg(&srv_addr);
|
||||
print_server_info();
|
||||
|
||||
if opt.console_enable {
|
||||
info!("console is enabled");
|
||||
debug!("console is enabled");
|
||||
let access_key = opt.access_key.clone();
|
||||
let secret_key = opt.secret_key.clone();
|
||||
let console_address = opt.console_address.clone();
|
||||
tokio::spawn(async move {
|
||||
let ep = if !opt.server_domains.is_empty() {
|
||||
format!("http://{}", opt.server_domains[0].clone())
|
||||
} else {
|
||||
format!("http://127.0.0.1:{}", server_port)
|
||||
};
|
||||
|
||||
console::start_static_file_server(&opt.console_address, &ep).await;
|
||||
console::start_static_file_server(&console_address, local_ip, &access_key, &secret_key).await;
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
10
rustfs/src/utils.rs
Normal file
10
rustfs/src/utils.rs
Normal file
@@ -0,0 +1,10 @@
|
||||
use local_ip_address;
|
||||
use std::net::IpAddr;
|
||||
|
||||
pub(crate) fn get_local_ip() -> Option<std::net::Ipv4Addr> {
|
||||
match local_ip_address::local_ip() {
|
||||
Ok(IpAddr::V4(ip)) => Some(ip),
|
||||
Err(_) => None,
|
||||
Ok(IpAddr::V6(_)) => todo!(),
|
||||
}
|
||||
}
|
||||
@@ -3,9 +3,20 @@ BIN=$1
|
||||
VOLUME=$2
|
||||
|
||||
chmod +x $BIN
|
||||
sudo mkdir -p $VOLUME
|
||||
mkdir -p $VOLUME
|
||||
|
||||
export RUST_LOG="rustfs=debug,ecstore=debug,s3s=debug,iam=debug"
|
||||
export RUST_BACKTRACE=full
|
||||
$BIN $VOLUME > /tmp/rustfs.log 2>&1 &
|
||||
|
||||
sudo nohup $BIN $VOLUME > /tmp/rustfs.log 2>&1 &
|
||||
sleep 10
|
||||
|
||||
export AWS_ACCESS_KEY_ID=rustfsadmin
|
||||
export AWS_SECRET_ACCESS_KEY=rustfsadmin
|
||||
export AWS_REGION=us-east-1
|
||||
export AWS_ENDPOINT_URL=http://localhost:9000
|
||||
export RUST_LOG="s3s_e2e=debug,s3s_test=info,s3s=debug"
|
||||
export RUST_BACKTRACE=full
|
||||
s3s-e2e
|
||||
|
||||
killall $BIN
|
||||
|
||||
32
scripts/run.bat
Normal file
32
scripts/run.bat
Normal file
@@ -0,0 +1,32 @@
|
||||
@echo off
|
||||
rem filepath: run.bat
|
||||
|
||||
if not defined SKIP_BUILD (
|
||||
cargo build -p rustfs --bins
|
||||
)
|
||||
|
||||
set current_dir=%cd%
|
||||
|
||||
if not exist .\target\volume\test mkdir .\target\volume\test
|
||||
|
||||
if not defined RUST_LOG (
|
||||
set RUST_BACKTRACE=1
|
||||
set RUST_LOG=rustfs=debug,ecstore=debug,s3s=debug,iam=debug
|
||||
)
|
||||
|
||||
rem set RUSTFS_ERASURE_SET_DRIVE_COUNT=5
|
||||
|
||||
rem set RUSTFS_STORAGE_CLASS_INLINE_BLOCK=512 KB
|
||||
|
||||
rem set RUSTFS_VOLUMES=.\target\volume\test{0...4}
|
||||
set RUSTFS_VOLUMES=.\target\volume\test
|
||||
set RUSTFS_ADDRESS=0.0.0.0:9000
|
||||
set RUSTFS_CONSOLE_ENABLE=true
|
||||
set RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9002
|
||||
rem set RUSTFS_SERVER_DOMAINS=localhost:9000
|
||||
|
||||
if not "%~1"=="" (
|
||||
set RUSTFS_VOLUMES=%~1
|
||||
)
|
||||
|
||||
cargo run --bin rustfs
|
||||
1
scripts/static.sh
Executable file
1
scripts/static.sh
Executable file
@@ -0,0 +1 @@
|
||||
curl -L "https://dl.rustfs.com/console/rustfs-console-latest.zip" -o tempfile.zip && unzip -o tempfile.zip -d ./rustfs/static && rm tempfile.zip
|
||||
Reference in New Issue
Block a user