diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index be58a46e..6b9dcfc4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -2,6 +2,8 @@ ## 📋 Code Quality Requirements +For instructions on setting up and running the local development environment, please see [Development Guide](docs/DEVELOPMENT.md). + ### 🔧 Code Formatting Rules **MANDATORY**: All code must be properly formatted before committing. This project enforces strict formatting standards to maintain code consistency and readability. diff --git a/Dockerfile.source b/Dockerfile.source index c4d9a430..73a628cb 100644 --- a/Dockerfile.source +++ b/Dockerfile.source @@ -72,7 +72,7 @@ COPY Cargo.toml Cargo.lock ./ # 2) workspace member manifests (adjust if workspace layout changes) COPY rustfs/Cargo.toml rustfs/Cargo.toml COPY crates/*/Cargo.toml crates/ -COPY cli/rustfs-gui/Cargo.toml cli/rustfs-gui/Cargo.toml + # Pre-fetch dependencies for better caching RUN --mount=type=cache,target=/usr/local/cargo/registry \ @@ -117,6 +117,49 @@ RUN --mount=type=cache,target=/usr/local/cargo/registry \ ;; \ esac +# ----------------------------- +# Development stage (keeps toolchain) +# ----------------------------- +FROM builder AS dev + +ARG BUILD_DATE +ARG VCS_REF + +LABEL name="RustFS (dev-source)" \ + maintainer="RustFS Team" \ + build-date="${BUILD_DATE}" \ + vcs-ref="${VCS_REF}" \ + description="RustFS - local development with Rust toolchain." + +# Install runtime dependencies that might be missing in partial builder +# (builder already has build-essential, lld, etc.) +WORKDIR /app + +ENV CARGO_INCREMENTAL=1 + +# Ensure we have the same default env vars available +ENV RUSTFS_ADDRESS=":9000" \ + RUSTFS_ACCESS_KEY="rustfsadmin" \ + RUSTFS_SECRET_KEY="rustfsadmin" \ + RUSTFS_CONSOLE_ENABLE="true" \ + RUSTFS_VOLUMES="/data" \ + RUST_LOG="warn" \ + RUSTFS_OBS_LOG_DIRECTORY="/logs" \ + RUSTFS_USERNAME="rustfs" \ + RUSTFS_GROUPNAME="rustfs" \ + RUSTFS_UID="1000" \ + RUSTFS_GID="1000" + +# Note: We don't COPY source here because we expect it to be mounted at /app +# We rely on cargo run to build and run +EXPOSE 9000 9001 + +COPY entrypoint.sh /entrypoint.sh +RUN chmod +x /entrypoint.sh + +ENTRYPOINT ["/entrypoint.sh"] +CMD ["cargo", "run", "--bin", "rustfs", "--"] + # ----------------------------- # Runtime stage (Ubuntu minimal) # ----------------------------- diff --git a/docker-compose.yml b/docker-compose.yml index 492803e3..2dd53a8c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -version: "3.8" - services: # RustFS main service rustfs: @@ -30,11 +28,11 @@ services: - "9000:9000" # S3 API port - "9001:9001" # Console port environment: - - RUSTFS_VOLUMES=/data/rustfs{0...3} # Define 4 storage volumes + - RUSTFS_VOLUMES=/data/rustfs{0..3} # Define 4 storage volumes - RUSTFS_ADDRESS=0.0.0.0:9000 - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_EXTERNAL_ADDRESS=:9000 # Same as internal since no port mapping + - RUSTFS_EXTERNAL_ADDRESS=:9000 # Same as internal since no port mapping - RUSTFS_CORS_ALLOWED_ORIGINS=* - RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS=* - RUSTFS_ACCESS_KEY=rustfsadmin @@ -43,9 +41,9 @@ services: - RUSTFS_TLS_PATH=/opt/tls - RUSTFS_OBS_ENDPOINT=http://otel-collector:4317 volumes: - - deploy/data/pro:/data - - deploy/logs:/app/logs - - deploy/data/certs/:/opt/tls # TLS configuration, you should create tls directory and put your tls files in it and then specify the path here + - ./deploy/data/pro:/data + - ./deploy/logs:/app/logs + - ./deploy/data/certs/:/opt/tls # TLS configuration, you should create tls directory and put your tls files in it and then specify the path here networks: - rustfs-network restart: unless-stopped @@ -61,7 +59,9 @@ services: retries: 3 start_period: 40s depends_on: - - otel-collector + otel-collector: + condition: service_started + required: false # Development environment rustfs-dev: @@ -70,16 +70,17 @@ services: build: context: . dockerfile: Dockerfile.source + target: dev # Pure development environment ports: - "9010:9000" # S3 API port - "9011:9001" # Console port environment: - - RUSTFS_VOLUMES=/data/rustfs{1...4} + - RUSTFS_VOLUMES=/data/rustfs{0..3} - RUSTFS_ADDRESS=0.0.0.0:9000 - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001 - RUSTFS_CONSOLE_ENABLE=true - - RUSTFS_EXTERNAL_ADDRESS=:9010 # External port mapping 9010 -> 9000 + - RUSTFS_EXTERNAL_ADDRESS=:9010 # External port mapping 9010 -> 9000 - RUSTFS_CORS_ALLOWED_ORIGINS=* - RUSTFS_CONSOLE_CORS_ALLOWED_ORIGINS=* - RUSTFS_ACCESS_KEY=devadmin @@ -88,7 +89,8 @@ services: - RUSTFS_OBS_LOG_DIRECTORY=/logs volumes: - .:/app # Mount source code to /app for development - - deploy/data/dev:/data + - cargo_registry:/usr/local/cargo/registry # Mount cargo registry to avoid re-downloading + - ./deploy/data/dev:/data networks: - rustfs-network restart: unless-stopped @@ -230,3 +232,5 @@ volumes: driver: local logs: driver: local + cargo_registry: + driver: local diff --git a/docs/DEVELOPMENT.md b/docs/DEVELOPMENT.md new file mode 100644 index 00000000..60767eb6 --- /dev/null +++ b/docs/DEVELOPMENT.md @@ -0,0 +1,71 @@ +# RustFS Local Development Guide + +This guide explains how to set up and run a local development environment for RustFS using Docker. This approach allows you to build and run the code from source in a consistent environment without needing to install the Rust toolchain on your host machine. + +## Prerequisites + +- [Docker](https://docs.docker.com/get-docker/) +- [Docker Compose](https://docs.docker.com/compose/install/) + +## Quick Start + +The development environment is configured as a Docker Compose profile named `dev`. + +### 1. Setup Console UI (Optional) + +If you want to use the Console UI, you must download the static assets first. The default source checkout does not include them. + +```bash +bash scripts/static.sh +``` + +### 2. Start the Environment + +To start the development container: + +```bash +docker compose --profile dev up -d rustfs-dev +``` + +**Note**: The first run will take some time (5-10 minutes) because it builds the docker image and compiles all Rust dependencies from source. Subsequent runs will be much faster. + +### 3. View Logs + +To follow the application logs: + +```bash +docker compose --profile dev logs -f rustfs-dev +``` + +### 4. Access the Services + +- **S3 API**: `http://localhost:9010` +- **Console UI**: `http://localhost:9011/rustfs/console/index.html` + +## Workflow + +### Making Changes +The source code from your local `rustfs` directory is mounted into the container at `/app`. You can edit files in your preferred IDE on your host machine. + +### Applying Changes +Since the application runs via `cargo run`, you need to restart the container to pick up changes. Thanks to incremental compilation, this is fast. + +```bash +docker compose --profile dev restart rustfs-dev +``` + +### Rebuilding Dependencies +If you modify `Cargo.toml` or `Cargo.lock`, you generally need to rebuild the Docker image to update the cached dependencies layer: + +```bash +docker compose --profile dev build rustfs-dev +``` + +## Troubleshooting + +### `VolumeNotFound` Error +If you see an error like `Error: Custom { kind: Other, error: VolumeNotFound }`, it means the `rustfs` binary was started without valid volume arguments. +The development image uses `entrypoint.sh` to parse the `RUSTFS_VOLUMES` environment variable (supporting `{N..M}` syntax), create the directories, and pass them to `cargo run`. Ensure your `RUSTFS_VOLUMES` variable is correctly formatted. + +### Slow Initial Build +This is expected. The `dev` stage in `Dockerfile.source` compiles all dependencies from scratch. Because the `/usr/local/cargo/registry` is mounted as a volume, these compiled artifacts are preserved between restarts, making future builds fast. diff --git a/entrypoint.sh b/entrypoint.sh index f9e605f6..e3466696 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -13,6 +13,8 @@ elif [ "${1#-}" != "$1" ]; then elif [ "$1" = "rustfs" ]; then shift set -- /usr/bin/rustfs "$@" +elif [ "$1" = "cargo" ]; then + : # Pass through cargo command as-is else set -- /usr/bin/rustfs "$@" fi @@ -22,8 +24,35 @@ DATA_VOLUMES="" process_data_volumes() { VOLUME_RAW="${RUSTFS_VOLUMES:-/data}" # Convert comma/tab to space - VOLUME_LIST=$(echo "$VOLUME_RAW" | tr ',\t' ' ') + VOLUME_LIST_RAW=$(echo "$VOLUME_RAW" | tr ',\t' ' ') + VOLUME_LIST="" + for vol in $VOLUME_LIST_RAW; do + # Helper to manually expand {N..M} since sh doesn't support it on variables + if echo "$vol" | grep -E -q "\{[0-9]+\.\.[0-9]+\}"; then + PREFIX=${vol%%\{*} + SUFFIX=${vol##*\}} + RANGE=${vol#*\{} + RANGE=${RANGE%\}} + START=${RANGE%%..*} + END=${RANGE##*..} + + # Check if START and END are numbers + if [ "$START" -eq "$START" ] 2>/dev/null && [ "$END" -eq "$END" 2>/dev/null ]; then + i=$START + while [ "$i" -le "$END" ]; do + VOLUME_LIST="$VOLUME_LIST ${PREFIX}${i}${SUFFIX}" + i=$((i+1)) + done + else + # Fallback if not numbers + VOLUME_LIST="$VOLUME_LIST $vol" + fi + else + VOLUME_LIST="$VOLUME_LIST $vol" + fi + done + for vol in $VOLUME_LIST; do case "$vol" in /*)