From a26ddf3aaa62ced268603fbb88091b14ace0b3b4 Mon Sep 17 00:00:00 2001 From: reatang Date: Sat, 17 Jan 2026 01:15:20 +0800 Subject: [PATCH] Resolving the issue of encryption parameters not being stored. --- .vscode/launch.json | 1 + rustfs/src/storage/ecfs.rs | 1 + rustfs/src/storage/sse.rs | 174 +++++++++++++++++----------------- scripts/generate-sse-keys.ps1 | 28 ++++++ scripts/generate-sse-keys.sh | 24 +++++ 5 files changed, 139 insertions(+), 89 deletions(-) create mode 100644 scripts/generate-sse-keys.ps1 create mode 100644 scripts/generate-sse-keys.sh diff --git a/.vscode/launch.json b/.vscode/launch.json index 62da1e91..e67d7fa8 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -114,6 +114,7 @@ // "RUSTFS_OBS_METRIC_ENDPOINT": "http://127.0.0.1:4318/v1/metrics", // default otlp http endpoint // "RUSTFS_OBS_LOG_ENDPOINT": "http://127.0.0.1:4318/v1/logs", // default otlp http endpoint // "RUSTFS_COMPRESS_ENABLE": "true", + "__RUSTFS_SSE_SIMPLE_CMK": "2dfNXGHlsEflGVCxb+5DIdGEl1sIvtwX+QfmYasi5QM=", "RUSTFS_CONSOLE_ADDRESS": "127.0.0.1:9001", "RUSTFS_OBS_LOG_DIRECTORY": "./target/logs", }, diff --git a/rustfs/src/storage/ecfs.rs b/rustfs/src/storage/ecfs.rs index 2c13eb0d..6dd5e9a5 100644 --- a/rustfs/src/storage/ecfs.rs +++ b/rustfs/src/storage/ecfs.rs @@ -3350,6 +3350,7 @@ impl S3 for FS { let mut reader = PutObjReader::new(reader); let mt2 = metadata.clone(); + opts.user_defined.extend(metadata); let repoptions = get_must_replicate_options(&mt2, "".to_string(), ReplicationStatusType::Empty, ReplicationType::Object, opts.clone()); diff --git a/rustfs/src/storage/sse.rs b/rustfs/src/storage/sse.rs index a16dad40..f37fbd34 100644 --- a/rustfs/src/storage/sse.rs +++ b/rustfs/src/storage/sse.rs @@ -664,18 +664,18 @@ async fn apply_managed_encryption_material( metadata.insert("x-rustfs-encryption-key".to_string(), BASE64_STANDARD.encode(&encryption_metadata.encrypted_data_key)); metadata.insert("x-rustfs-encryption-iv".to_string(), BASE64_STANDARD.encode(&encryption_metadata.iv)); metadata.insert("x-rustfs-encryption-algorithm".to_string(), encryption_metadata.algorithm.clone()); + metadata.insert("x-amz-server-side-encryption".to_string(), algorithm_str.to_string()); + + // if kms_key is changed, we need to update the metadata + if kms_key_id.is_none() { + metadata.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), kms_key_to_use.clone()); + } } metadata.insert( "x-rustfs-encryption-original-size".to_string(), encryption_metadata.original_size.to_string(), ); - metadata.insert("x-amz-server-side-encryption".to_string(), algorithm_str.to_string()); - - // if kms_key is changed, we need to update the metadata - if kms_key_id.is_none() { - metadata.insert("x-amz-server-side-encryption-aws-kms-key-id".to_string(), kms_key_to_use.clone()); - } // Handle part-specific nonce if needed let nonce = if let Some(part_num) = part_number { @@ -906,63 +906,80 @@ impl SseDekProvider for KmsSseDekProvider { // Test/Simple DEK Provider // ============================================================================ -// Implement a simple SSE DEK provider for testing purposes +/// Simple SSE DEK provider for testing purposes +/// +/// This provider reads a single 32-byte customer master key (CMK) from the +/// `__RUSTFS_SSE_SIMPLE_CMK` environment variable. The key must be base64-encoded. +/// +/// # Environment Variable Format +/// +/// ```text +/// __RUSTFS_SSE_SIMPLE_CMK= +/// ``` +/// +/// Example: +/// ```bash +/// export __RUSTFS_SSE_SIMPLE_CMK="AKHul86TBMMJ3+VrGlh9X3dHJsOtSXOXHOODPwmAnOo=" +/// ``` +/// +/// # Key Generation +/// +/// Use the provided script to generate a valid key: +/// ```bash +/// # Windows +/// .\scripts\generate-sse-keys.ps1 +/// +/// # Linux/Unix/macOS +/// ./scripts/generate-sse-keys.sh +/// ``` struct SimpleSseDekProvider { - cmk_ids: HashMap, + master_key: [u8; 32], } -// __RUSTFS_SSE_SIMPLE_CMK_ID format: key-id1:base64_key1,key-id2:base64_key2,... - impl SimpleSseDekProvider { - /// Create a SimpleSseDekProvider with predefined keys (for testing) + /// Create a SimpleSseDekProvider with a predefined key (for testing) #[cfg(test)] - pub fn new_with_keys(cmk_ids: HashMap) -> Self { - Self { cmk_ids } + pub fn new_with_key(master_key: [u8; 32]) -> Self { + Self { master_key } } pub fn new() -> Self { - let cmk_id = std::env::var("__RUSTFS_SSE_SIMPLE_CMK_ID").unwrap_or_else(|_| "".to_string()); + let cmk_value = std::env::var("__RUSTFS_SSE_SIMPLE_CMK").unwrap_or_else(|_| "".to_string()); - let cmk_ids: HashMap = cmk_id - .split(',') - .filter_map(|pair| { - let parts: Vec<&str> = pair.split(':').collect(); - if parts.len() == 2 { - let key_name = parts[0]; - match BASE64_STANDARD.decode(parts[1]) { - Ok(v) => { - let decoded_len = v.len(); - match v.try_into() { - Ok(arr) => { - println!("✓ Successfully loaded CMK: {}", key_name); - Some((key_name.to_string(), arr)) - } - Err(_) => { - eprintln!( - "✗ Failed to load CMK '{}': decoded key is not 32 bytes (got {} bytes)", - key_name, decoded_len - ); - None - } - } + let master_key = if !cmk_value.is_empty() { + match BASE64_STANDARD.decode(cmk_value.trim()) { + Ok(v) => { + let decoded_len = v.len(); + match v.try_into() { + Ok(arr) => { + println!("✓ Successfully loaded master key (32 bytes)"); + arr } - Err(e) => { - eprintln!("✗ Failed to load CMK '{}': invalid base64 encoding: {}", key_name, e); - None + Err(_) => { + eprintln!( + "✗ Failed to load master key: decoded key is not 32 bytes (got {} bytes)", + decoded_len + ); + [0u8; 32] } } - } else { - eprintln!("✗ Invalid CMK format in '{}': expected 'name:base64_key'", pair); - None } - }) - .collect(); + Err(e) => { + eprintln!("✗ Failed to load master key: invalid base64 encoding: {}", e); + [0u8; 32] + } + } + } else { + [0u8; 32] + }; - if cmk_ids.is_empty() { - eprintln!("⚠️ WARNING: No valid CMK keys loaded! All encryption operations will fail."); + if master_key == [0u8; 32] { + eprintln!("✗ Failed to load master key: no valid master key loaded! All encryption operations will fail."); + eprintln!(" Set __RUSTFS_SSE_SIMPLE_CMK environment variable to a base64-encoded 32-byte key."); + std::process::exit(1); } - Self { cmk_ids } + Self { master_key: master_key } } // Simple encryption of DEK @@ -1016,13 +1033,7 @@ impl SimpleSseDekProvider { #[async_trait] impl SseDekProvider for SimpleSseDekProvider { - async fn generate_sse_dek(&self, _bucket: &str, _key: &str, kms_key_id: &str) -> Result<(DataKey, Vec), ApiError> { - let cmk_value = self - .cmk_ids - .get(kms_key_id) - .ok_or_else(|| ApiError::from(StorageError::other(format!("CMK ID not found: {}", kms_key_id))))? - .clone(); - + async fn generate_sse_dek(&self, _bucket: &str, _key: &str, _kms_key_id: &str) -> Result<(DataKey, Vec), ApiError> { // Generate a 32-byte array as data key let mut dek = [0u8; 32]; rand::rng().fill_bytes(&mut dek); @@ -1031,8 +1042,8 @@ impl SseDekProvider for SimpleSseDekProvider { let mut nonce = [0u8; 12]; rand::rng().fill_bytes(&mut nonce); - // Encrypt data key - let encrypted_dek = Self::encrypt_dek(dek, cmk_value)?; + // Encrypt data key with master key + let encrypted_dek = Self::encrypt_dek(dek, self.master_key)?; // Return data key and IV Ok(( @@ -1044,18 +1055,11 @@ impl SseDekProvider for SimpleSseDekProvider { )) } - async fn decrypt_sse_dek(&self, encrypted_dek: &[u8], kms_key_id: &str) -> Result<[u8; 32], ApiError> { - // Verify CMK ID exists (for testing purposes) - let cmk_value = self - .cmk_ids - .get(kms_key_id) - .ok_or_else(|| ApiError::from(StorageError::other(format!("CMK ID not found: {}", kms_key_id))))? - .clone(); - - // Decrypt data key + async fn decrypt_sse_dek(&self, encrypted_dek: &[u8], _kms_key_id: &str) -> Result<[u8; 32], ApiError> { + // Decrypt data key with master key let encrypted_dek_str = std::str::from_utf8(encrypted_dek) .map_err(|_| ApiError::from(StorageError::other("Invalid UTF-8 in encrypted DEK")))?; - let dek = Self::decrypt_dek(encrypted_dek_str, cmk_value)?; + let dek = Self::decrypt_dek(encrypted_dek_str, self.master_key)?; Ok(dek) } } @@ -1070,7 +1074,7 @@ static GLOBAL_SSE_DEK_PROVIDER: OnceLock> = OnceLock::ne /// Get or initialize the global SSE DEK provider /// /// Factory function that automatically selects the appropriate provider: -/// - If `__RUSTFS_SSE_SIMPLE_CMK_ID` environment variable exists: use SimpleSseDekProvider (test mode) +/// - If `__RUSTFS_SSE_SIMPLE_CMK` environment variable exists: use SimpleSseDekProvider (test mode) /// - Otherwise: use KmsSseDekProvider (production mode with real KMS) /// /// # Returns @@ -1090,8 +1094,8 @@ pub async fn get_sse_dek_provider() -> Result, ApiError> } // Determine provider based on environment variable - let provider: Arc = if std::env::var("__RUSTFS_SSE_SIMPLE_CMK_ID").is_ok() { - debug!("Using SimpleSseDekProvider (test mode) based on __RUSTFS_SSE_SIMPLE_CMK_ID"); + let provider: Arc = if std::env::var("__RUSTFS_SSE_SIMPLE_CMK").is_ok() { + debug!("Using SimpleSseDekProvider (test mode) based on __RUSTFS_SSE_SIMPLE_CMK"); Arc::new(SimpleSseDekProvider::new()) } else { debug!("Using KmsSseDekProvider (production mode)"); @@ -1613,15 +1617,13 @@ mod tests { use std::io::Cursor; use tokio::io::AsyncReadExt; - // 1. Setup: Create SimpleSseDekProvider with test CMK - let mut test_keys = HashMap::new(); - test_keys.insert("test-key".to_string(), [42u8; 32]); - let provider = SimpleSseDekProvider::new_with_keys(test_keys); + // 1. Setup: Create SimpleSseDekProvider with test master key + let provider = SimpleSseDekProvider::new_with_key([42u8; 32]); // 2. Generate a data encryption key let bucket = "test-bucket"; let key = "test-key"; - let kms_key_id = "test-key"; // Must match the key in SimpleSseDekProvider::new() + let kms_key_id = "default"; // Key ID is ignored in simple provider let (data_key, _encrypted_dek) = provider .generate_sse_dek(bucket, key, kms_key_id) @@ -1682,14 +1684,12 @@ mod tests { use std::io::Cursor; use tokio::io::AsyncReadExt; - // 1. Setup: Create SimpleSseDekProvider with test CMK - let mut test_keys = HashMap::new(); - test_keys.insert("test-key".to_string(), [42u8; 32]); - let provider = SimpleSseDekProvider::new_with_keys(test_keys); + // 1. Setup: Create SimpleSseDekProvider with test master key + let provider = SimpleSseDekProvider::new_with_key([42u8; 32]); let bucket = "test-bucket"; let key = "test-key-large"; - let kms_key_id = "test-key"; + let kms_key_id = "default"; let (data_key, _encrypted_dek) = provider .generate_sse_dek(bucket, key, kms_key_id) @@ -1735,14 +1735,12 @@ mod tests { use std::io::Cursor; use tokio::io::AsyncReadExt; - // 1. Setup: Create SimpleSseDekProvider with test CMK - let mut test_keys = HashMap::new(); - test_keys.insert("test-key".to_string(), [42u8; 32]); - let provider = SimpleSseDekProvider::new_with_keys(test_keys); + // 1. Setup: Create SimpleSseDekProvider with test master key + let provider = SimpleSseDekProvider::new_with_key([42u8; 32]); let bucket = "test-bucket"; let key = "test-key"; - let kms_key_id = "test-key"; + let kms_key_id = "default"; // Generate two different keys (with different nonces) let (data_key1, _) = provider @@ -1787,14 +1785,12 @@ mod tests { use std::io::Cursor; use tokio::io::AsyncReadExt; - // 1. Setup: Create SimpleSseDekProvider with test CMK - let mut test_keys = HashMap::new(); - test_keys.insert("test-key".to_string(), [42u8; 32]); - let provider = SimpleSseDekProvider::new_with_keys(test_keys); + // 1. Setup: Create SimpleSseDekProvider with test master key + let provider = SimpleSseDekProvider::new_with_key([42u8; 32]); let bucket = "test-bucket"; let key = "test-key"; - let kms_key_id = "test-key"; + let kms_key_id = "default"; // 1. Generate DEK and get encrypted DEK let (data_key, encrypted_dek) = provider diff --git a/scripts/generate-sse-keys.ps1 b/scripts/generate-sse-keys.ps1 new file mode 100644 index 00000000..743b1e12 --- /dev/null +++ b/scripts/generate-sse-keys.ps1 @@ -0,0 +1,28 @@ +# Generate SSE encryption key (32 bytes base64 encoded) +# For testing with __RUSTFS_SSE_SIMPLE_CMK environment variable +# Usage: .\generate-sse-keys.ps1 + +# Function to generate a random 256-bit key and encode it in base64 +function Generate-Base64Key { + # Generate 32 bytes (256 bits) of random data + $bytes = New-Object byte[] 32 + $rng = [System.Security.Cryptography.RandomNumberGenerator]::Create() + $rng.GetBytes($bytes) + $rng.Dispose() + + # Convert to base64 + return [Convert]::ToBase64String($bytes) +} + +# Generate key +$base64Key = Generate-Base64Key + +# Output result +Write-Host "" +Write-Host "Generated SSE encryption key (32 bytes, base64 encoded):" -ForegroundColor Green +Write-Host "" +Write-Host $base64Key -ForegroundColor Yellow +Write-Host "" +Write-Host "You can use this in your environment variable:" -ForegroundColor Cyan +Write-Host "`$env:__RUSTFS_SSE_SIMPLE_CMK=`"$base64Key`"" -ForegroundColor White +Write-Host "" diff --git a/scripts/generate-sse-keys.sh b/scripts/generate-sse-keys.sh new file mode 100644 index 00000000..48a2dc3b --- /dev/null +++ b/scripts/generate-sse-keys.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Generate SSE encryption key (32 bytes base64 encoded) +# For testing with __RUSTFS_SSE_SIMPLE_CMK environment variable +# Usage: ./generate-sse-keys.sh + +set -euo pipefail + +# Function to generate a random 256-bit key and encode it in base64 +generate_base64_key() { + # Generate 32 bytes (256 bits) of random data and base64 encode it + openssl rand -base64 32 | tr -d '\n' +} + +# Generate key +base64_key=$(generate_base64_key) + +# Output result +echo "Generated SSE encryption key (32 bytes, base64 encoded):" +echo "" +echo "$base64_key" +echo "" +echo "You can use this in your environment variable:" +echo "export __RUSTFS_SSE_SIMPLE_CMK=\"$base64_key\"" +