mirror of
https://github.com/rustfs/rustfs.git
synced 2026-03-17 14:24:08 +00:00
refactor(obs): optimize logging with custom RollingAppender and improved cleanup (#2151)
Signed-off-by: houseme <housemecn@gmail.com> Signed-off-by: heihutu <30542132+heihutu@users.noreply.github.com> Co-authored-by: Copilot <198982749+Copilot@users.noreply.github.com> Co-authored-by: houseme <4829346+houseme@users.noreply.github.com> Co-authored-by: heihutu <30542132+heihutu@users.noreply.github.com> Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com>
This commit is contained in:
2
.github/workflows/nix-flake-update.yml
vendored
2
.github/workflows/nix-flake-update.yml
vendored
@@ -31,7 +31,7 @@ jobs:
|
||||
update-flake:
|
||||
name: Update flake.lock
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
4
.github/workflows/nix.yml
vendored
4
.github/workflows/nix.yml
vendored
@@ -43,7 +43,7 @@ jobs:
|
||||
nix-validation:
|
||||
name: Nix Build & Check
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 30
|
||||
timeout-minutes: 45
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
|
||||
- name: Setup Magic Nix Cache
|
||||
uses: DeterminateSystems/magic-nix-cache-action@v13
|
||||
|
||||
|
||||
- name: Setup Flake Checker
|
||||
uses: DeterminateSystems/flake-checker-action@v12
|
||||
|
||||
|
||||
279
Cargo.lock
generated
279
Cargo.lock
generated
@@ -148,9 +148,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299"
|
||||
|
||||
[[package]]
|
||||
name = "anstream"
|
||||
version = "0.6.21"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
|
||||
checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"anstyle-parse",
|
||||
@@ -169,9 +169,9 @@ checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
|
||||
|
||||
[[package]]
|
||||
name = "anstyle-parse"
|
||||
version = "0.2.7"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
|
||||
checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e"
|
||||
dependencies = [
|
||||
"utf8parse",
|
||||
]
|
||||
@@ -706,9 +706,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "aws-sdk-s3"
|
||||
version = "1.125.0"
|
||||
version = "1.126.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "223f5c95650d9557925a91f4c2db3def189e8f659452134a29e5cd2d37d708ed"
|
||||
checksum = "7878050a2321d215eec9db8be09f8db59418b53860ae86cc7042b4094d6cb2bb"
|
||||
dependencies = [
|
||||
"aws-credential-types",
|
||||
"aws-runtime",
|
||||
@@ -1470,7 +1470,7 @@ version = "0.4.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad"
|
||||
dependencies = [
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"inout 0.1.4",
|
||||
]
|
||||
|
||||
@@ -1487,9 +1487,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap"
|
||||
version = "4.5.60"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a"
|
||||
checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351"
|
||||
dependencies = [
|
||||
"clap_builder",
|
||||
"clap_derive",
|
||||
@@ -1497,9 +1497,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_builder"
|
||||
version = "4.5.60"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876"
|
||||
checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f"
|
||||
dependencies = [
|
||||
"anstream",
|
||||
"anstyle",
|
||||
@@ -1509,9 +1509,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_derive"
|
||||
version = "4.5.55"
|
||||
version = "4.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5"
|
||||
checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a"
|
||||
dependencies = [
|
||||
"heck",
|
||||
"proc-macro2",
|
||||
@@ -1521,9 +1521,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "clap_lex"
|
||||
version = "1.0.0"
|
||||
version = "1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a822ea5bc7590f9d40f1ba12c0dc3c2760f3482c6984db1573ad11031420831"
|
||||
checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9"
|
||||
|
||||
[[package]]
|
||||
name = "clocksource"
|
||||
@@ -1927,9 +1927,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-bigint"
|
||||
version = "0.7.0-rc.28"
|
||||
version = "0.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "96dacf199529fb801ae62a9aafdc01b189e9504c0d1ee1512a4c16bcd8666a93"
|
||||
checksum = "9fde2467e74147f492aebb834985186b2c74761927b8b9b3bd303bcb2e72199d"
|
||||
dependencies = [
|
||||
"cpubits",
|
||||
"ctutils",
|
||||
@@ -1941,9 +1941,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-common"
|
||||
version = "0.1.7"
|
||||
version = "0.1.6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"
|
||||
checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3"
|
||||
dependencies = [
|
||||
"generic-array",
|
||||
"typenum",
|
||||
@@ -1962,11 +1962,11 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "crypto-primes"
|
||||
version = "0.7.0-pre.9"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6081ce8b60c0e533e2bba42771b94eb6149052115f4179744d5779883dc98583"
|
||||
checksum = "21f41f23de7d24cdbda7f0c4d9c0351f99a4ceb258ef30e5c1927af8987ffe5a"
|
||||
dependencies = [
|
||||
"crypto-bigint 0.7.0-rc.28",
|
||||
"crypto-bigint 0.7.1",
|
||||
"libm",
|
||||
"rand_core 0.10.0",
|
||||
]
|
||||
@@ -2204,9 +2204,9 @@ checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea"
|
||||
|
||||
[[package]]
|
||||
name = "datafusion"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "503f1f4a9060ae6e650d3dff5dc7a21266fea1302d890768d45b4b28586e830f"
|
||||
checksum = "ea28305c211e3541c9cfcf06a23d0d8c7c824b4502ed1fdf0a6ff4ad24ee531c"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-schema",
|
||||
@@ -2259,9 +2259,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-catalog"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "14417a3ee4ae3d092b56cd6c1d32e8ff3e2c9ec130ecb2276ec91c89fd599399"
|
||||
checksum = "78ab99b6df5f60a6ddbc515e4c05caee1192d395cf3cb67ce5d1c17e3c9b9b74"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2284,9 +2284,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-catalog-listing"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9d0eba824adb45a4b3ac6f0251d40df3f6a9382371cad136f4f14ac9ebc6bc10"
|
||||
checksum = "77ae3d14912c0d779ada98d30dc60f3244f3c26c2446b87394629ea5c076a31c"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2307,9 +2307,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-common"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0039deefbd00c56adf5168b7ca58568fb058e4ba4c5a03b09f8be371b4e434b6"
|
||||
checksum = "ea2df29b9592a5d55b8238eaf67d2f21963d5a08cd1a8b7670134405206caabd"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow",
|
||||
@@ -2331,9 +2331,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-common-runtime"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ec7e3e60b813048331f8fb9673583173e5d2dd8fef862834ee871fc98b57ca7"
|
||||
checksum = "42639baa0049d5fffd7e283504b9b5e7b9b2e7a2dea476eed60ab0d40d999b85"
|
||||
dependencies = [
|
||||
"futures",
|
||||
"log",
|
||||
@@ -2342,9 +2342,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "802068957f620302ecf05f84ff4019601aeafd36f5f3f1334984af2e34265129"
|
||||
checksum = "25951b617bb22a9619e1520450590cb2004bfcad10bcb396b961f4a1a10dcec5"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-compression",
|
||||
@@ -2377,9 +2377,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-arrow"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "90fc387d5067c62d494a6647d29c5ad4fcdd5a6e50ab4ea1d2568caa2d66f2cc"
|
||||
checksum = "dc0b28226960ba99c50d78ac6f736ebe09eb5cb3bb9bb58194266278000ca41f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ipc",
|
||||
@@ -2401,9 +2401,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-csv"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "efd5e20579bb6c8bd4e6c620253972fb723822030c280dd6aa047f660d09eeba"
|
||||
checksum = "f538b57b052a678b1ce860181c65d3ace5a8486312dc50b41c01dd585a773a51"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2424,9 +2424,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-json"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c0788b0d48fcef31880a02013ea3cc18e5a4e0eacc3b0abdd2cd0597b99dc96e"
|
||||
checksum = "89fbc1d32b1b03c9734e27c0c5f041232b68621c8455f22769838634750a196c"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2446,9 +2446,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-datasource-parquet"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "66639b70f1f363f5f0950733170100e588f1acfacac90c1894e231194aa35957"
|
||||
checksum = "203271d31fe5613a5943181db70ec98162121d1de94a9a300d5e5f19f9500a32"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2476,15 +2476,15 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-doc"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e44b41f3e8267c6cf3eec982d63f34db9f1dd5f30abfd2e1f124f0871708952e"
|
||||
checksum = "5b6450dc702b3d39e8ced54c3356abb453bd2f3cea86d90d555a4b92f7a38462"
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-execution"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9e456f60e5d38db45335e84617006d90af14a8c8c5b8e959add708b2daaa0e2c"
|
||||
checksum = "e66a02fa601de49da5181dbdcf904a18b16a184db2b31f5e5534552ea2d5e660"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2503,9 +2503,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-expr"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6507c719804265a58043134580c1c20767e7c23ba450724393f03ec982769ad9"
|
||||
checksum = "cdf59a9b308a1a07dc2eb2f85e6366bc0226dc390b40f3aa0a72d79f1cfe2465"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2526,9 +2526,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-expr-common"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a413caa9c5885072b539337aed68488f0291653e8edd7d676c92df2480f6cab0"
|
||||
checksum = "bd99eac4c6538c708638db43e7a3bd88e0e57955ddb722d420fb9a6d38dfc28f"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -2539,9 +2539,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "189256495dc9cbbb8e20dbcf161f60422e628d201a78df8207e44bd4baefadb6"
|
||||
checksum = "11aa2c492ac046397b36d57c62a72982aad306495bbcbcdbcabd424d4a2fe245"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-buffer",
|
||||
@@ -2570,9 +2570,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-aggregate"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12e73dfee4cd67c4a507ffff4c5a711d39983adf544adbc09c09bf06f789f413"
|
||||
checksum = "325a00081898945d48d6194d9ca26120e523c993be3bb7c084061a5a2a72e787"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow",
|
||||
@@ -2591,9 +2591,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-aggregate-common"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "87727bd9e65f4f9ac6d608c9810b7da9eaa3b18b26a4a4b76520592d49020acf"
|
||||
checksum = "809bbcb1e0dbec5d0ce30d493d135aea7564f1ba4550395f7f94321223df2dae"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow",
|
||||
@@ -2604,9 +2604,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-nested"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2e5ef761359224b7c2b5a1bfad6296ac63225f8583d08ad18af9ba1a89ac3887"
|
||||
checksum = "29ebaa5d7024ef45973e0a7db1e9aeaa647936496f4d4061c0448f23d77d6320"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"arrow-ord",
|
||||
@@ -2627,9 +2627,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-table"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3b17dac25dfda2d2a90ff0ad1c054a11fb1523766226bec6e9bd8c410daee2ae"
|
||||
checksum = "60eab6f39df9ee49a2c7fa38eddc01fa0086ee31b29c7d19f38e72f479609752"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"async-trait",
|
||||
@@ -2643,9 +2643,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-window"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c594a29ddb22cbdbce500e4d99b5b2392c5cecb4c1086298b41d1ffec14dbb77"
|
||||
checksum = "e00b2c15e342a90e65a846199c9e49293dd09fe1bcd63d8be2544604892f7eb8"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -2661,9 +2661,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-functions-window-common"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9aa1b15ed81c7543f62264a30dd49dec4b1b0b698053b968f53be32dfba4f729"
|
||||
checksum = "493e2e1d1f4753dfc139a5213f1b5d0b97eea46a82d9bda3c7908aa96981b74b"
|
||||
dependencies = [
|
||||
"datafusion-common",
|
||||
"datafusion-physical-expr-common",
|
||||
@@ -2671,9 +2671,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-macros"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c00c31c4795597aa25b74cab5174ac07a53051f27ce1e011ecaffa9eaeecef81"
|
||||
checksum = "ba01c55ade8278a791b429f7bf5cb1de64de587a342d084b18245edfae7096e2"
|
||||
dependencies = [
|
||||
"datafusion-doc",
|
||||
"quote",
|
||||
@@ -2682,9 +2682,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-optimizer"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "80ccf60767c09302b2e0fc3afebb3761a6d508d07316fab8c5e93312728a21bb"
|
||||
checksum = "a80c6dfbba6a2163a9507f6353ac78c69d8deb26232c9e419160e58ff7c3e047"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"chrono",
|
||||
@@ -2702,9 +2702,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c64b7f277556944e4edd3558da01d9e9ff9f5416f1c0aa7fee088e57bd141a7e"
|
||||
checksum = "5d3a86264bb9163e7360b6622e789bc7fcbb43672e78a8493f0bc369a41a57c6"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow",
|
||||
@@ -2726,9 +2726,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr-adapter"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b7abaee372ea2d19c016ee9ef8629c4415257d291cdd152bc7f0b75f28af1b63"
|
||||
checksum = "3f5e00e524ac33500be6c5eeac940bd3f6b984ba9b7df0cd5f6c34a8a2cc4d6b"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -2741,9 +2741,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-expr-common"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42237efe621f92adc22d111b531fdbc2cc38ca9b5e02327535628fb103ae2157"
|
||||
checksum = "2ae769ea5d688b4e74e9be5cad6f9d9f295b540825355868a3ab942380dd97ce"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow",
|
||||
@@ -2758,9 +2758,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-optimizer"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fd093498bd1319c6e5c76e9dfa905e78486f01b34579ce97f2e3a49f84c37fac"
|
||||
checksum = "f3588753ab2b47b0e43cd823fe5e7944df6734dabd6dafb72e2cc1c2a22f1944"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -2777,9 +2777,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-physical-plan"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7cbe61b12daf81a9f20ba03bd3541165d51f86e004ef37426b11881330eed261"
|
||||
checksum = "79949cbb109c2a45c527bfe0d956b9f2916807c05d4d2e66f3fd0af827ac2b61"
|
||||
dependencies = [
|
||||
"ahash",
|
||||
"arrow",
|
||||
@@ -2808,9 +2808,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-pruning"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0124331116db7f79df92ebfd2c3b11a8f90240f253555c9bb084f10b6fecf1dd"
|
||||
checksum = "6434e2ee8a39d04b95fed688ff34dc251af6e4a0c2e1714716b6e3846690d589"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"datafusion-common",
|
||||
@@ -2825,9 +2825,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-session"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1673e3c58ba618a6ea0568672f00664087b8982c581e9afd5aa6c3c79c9b431f"
|
||||
checksum = "c91efb8302b4877d499c37e9a71886b90236ab27d9cc42fd51112febf341abd6"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"datafusion-common",
|
||||
@@ -2839,9 +2839,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "datafusion-sql"
|
||||
version = "52.2.0"
|
||||
version = "52.3.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5272d256dab5347bb39d2040589f45d8c6b715b27edcb5fffe88cc8b9c3909cb"
|
||||
checksum = "3f01eef7bcf4d00e87305b55f1b75792384e130fe0258bac02cd48378ae5ff87"
|
||||
dependencies = [
|
||||
"arrow",
|
||||
"bigdecimal",
|
||||
@@ -3025,7 +3025,7 @@ checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292"
|
||||
dependencies = [
|
||||
"block-buffer 0.10.4",
|
||||
"const-oid 0.9.6",
|
||||
"crypto-common 0.1.7",
|
||||
"crypto-common 0.1.6",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
@@ -3606,9 +3606,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "generic-array"
|
||||
version = "0.14.7"
|
||||
version = "0.14.9"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a"
|
||||
checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
"version_check",
|
||||
@@ -3703,15 +3703,18 @@ checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280"
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-auth"
|
||||
version = "1.6.0"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "36ad774d41426ab205eeec577540f209a5485c366814dd5c89a7e3018fe84e7c"
|
||||
checksum = "1f83bc5c208df4a6b38ad2a8d2b01c0d377811f9efe9b0733171f28dd74db9c3"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"aws-lc-rs",
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
"chrono",
|
||||
"google-cloud-gax",
|
||||
"hex",
|
||||
"hmac 0.12.1",
|
||||
"http 1.4.0",
|
||||
"jsonwebtoken",
|
||||
"reqwest 0.13.2",
|
||||
@@ -3720,16 +3723,18 @@ dependencies = [
|
||||
"rustls-pki-types",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"sha2 0.10.9",
|
||||
"thiserror 2.0.18",
|
||||
"time",
|
||||
"tokio",
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-gax"
|
||||
version = "1.7.0"
|
||||
version = "1.8.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2973715fe664ecb0d883926c8b5f66cb9d52a44add1d0be1cad1907d832bf0af"
|
||||
checksum = "188909653b7c484e43695325c0324804b5645d568f8d2e4c8a6f520231d50956"
|
||||
dependencies = [
|
||||
"base64 0.22.1",
|
||||
"bytes",
|
||||
@@ -3747,9 +3752,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-gax-internal"
|
||||
version = "0.7.9"
|
||||
version = "0.7.10"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "598e5ffec2c1c9b43e83847b2badc0f128a03e541b75f1ecd8acf0a2605c40cf"
|
||||
checksum = "7395094c1dc7284155a48530aa1a49d52c876ce1b392dfcdf7e6b32540d042a2"
|
||||
dependencies = [
|
||||
"bytes",
|
||||
"futures",
|
||||
@@ -3761,6 +3766,8 @@ dependencies = [
|
||||
"http-body 1.0.1",
|
||||
"http-body-util",
|
||||
"hyper",
|
||||
"lazy_static",
|
||||
"opentelemetry",
|
||||
"opentelemetry-semantic-conventions",
|
||||
"percent-encoding",
|
||||
"pin-project",
|
||||
@@ -3781,9 +3788,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-iam-v1"
|
||||
version = "1.5.0"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b41025359d1f52c24966c24d5ecb7f91198ded13c9cbf57c2d2e735dae93c814"
|
||||
checksum = "f436945fb3c581a3ef32f37e47756690006de7177934c6405cc0d4db799c8975"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -3800,9 +3807,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-longrunning"
|
||||
version = "1.6.0"
|
||||
version = "1.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "daafe6c2859976ca51571db6c7bbf96bdce0122e81e6854a6c62554c4f63bae8"
|
||||
checksum = "767db5d07fff5361d01058764e724c4335b9899aa6c973c3bb2ae8d3bd4eacd8"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"bytes",
|
||||
@@ -3846,9 +3853,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "google-cloud-storage"
|
||||
version = "1.8.0"
|
||||
version = "1.9.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ce225eccfece751251e07ac787199cfc53bc2b5d60ac0570f58404507ceb309"
|
||||
checksum = "1be397108904bd24fb7b1518b68fc26a70589f3718fa8c47e9af5f09c4fc6e88"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -4168,9 +4175,9 @@ checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
|
||||
|
||||
[[package]]
|
||||
name = "hybrid-array"
|
||||
version = "0.4.7"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1b229d73f5803b562cc26e4da0396c8610a4ee209f4fac8fa4f8d709166dc45"
|
||||
checksum = "8655f91cd07f2b9d0c24137bd650fe69617773435ee5ec83022377777ce65ef1"
|
||||
dependencies = [
|
||||
"typenum",
|
||||
]
|
||||
@@ -5501,18 +5508,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "objc2-core-foundation"
|
||||
version = "0.3.1"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166"
|
||||
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-io-kit"
|
||||
version = "0.3.1"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a"
|
||||
checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"objc2-core-foundation",
|
||||
@@ -5572,9 +5579,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "once_cell"
|
||||
version = "1.21.3"
|
||||
version = "1.21.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
|
||||
checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50"
|
||||
|
||||
[[package]]
|
||||
name = "once_cell_polyfill"
|
||||
@@ -5865,9 +5872,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "password-hash"
|
||||
version = "0.6.0-rc.12"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5fa9e3d1c7b6f3e230b60fa44adc855cb8e24eede37236621f2cc1940d95564f"
|
||||
checksum = "ccbd25f71dd5249dba9ed843d52500c8757a25511560d01a94f4abf56b52a1d5"
|
||||
dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"phc",
|
||||
@@ -5990,13 +5997,13 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "phc"
|
||||
version = "0.6.0-rc.1"
|
||||
version = "0.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "71d390c5fe8d102c2c18ff39f1e72b9ad5996de282c2d831b0312f56910f5508"
|
||||
checksum = "44dc769b75f93afdddd8c7fa12d685292ddeff1e66f7f0f3a234cf1818afe892"
|
||||
dependencies = [
|
||||
"base64ct",
|
||||
"ctutils",
|
||||
"getrandom 0.4.2",
|
||||
"subtle",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6474,9 +6481,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "pyroscope"
|
||||
version = "1.0.2"
|
||||
version = "2.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f9a34d9b74a7a13140e626ed568fc984f520747ed6cf9e83bb532bb2f311ec75"
|
||||
checksum = "85eebd4bcbf45db75f67d2ba20ea0207bd111d2029c07a7db3229289173d4387"
|
||||
dependencies = [
|
||||
"lazy_static",
|
||||
"libc",
|
||||
@@ -6982,9 +6989,9 @@ checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422"
|
||||
|
||||
[[package]]
|
||||
name = "rmcp"
|
||||
version = "1.1.0"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2cb14cb9278a12eae884c9f3c0cfeca2cc28f361211206424a1d7abed95f090"
|
||||
checksum = "ba6b9d2f0efe2258b23767f1f9e0054cfbcac9c2d6f81a031214143096d7864f"
|
||||
dependencies = [
|
||||
"async-trait",
|
||||
"base64 0.22.1",
|
||||
@@ -7004,9 +7011,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rmcp-macros"
|
||||
version = "1.1.0"
|
||||
version = "1.2.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6a02ea81d9482b07e1fe156ac7cf98b6823d51fb84531936a5e1cbb4eec31ad5"
|
||||
checksum = "ab9d95d7ed26ad8306352b0d5f05b593222b272790564589790d210aa15caa9e"
|
||||
dependencies = [
|
||||
"darling 0.23.0",
|
||||
"proc-macro2",
|
||||
@@ -7056,12 +7063,12 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "rsa"
|
||||
version = "0.10.0-rc.16"
|
||||
version = "0.10.0-rc.17"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fb9fd8c1edd9e6a2693623baf0fe77ff05ce022a5d7746900ffc38a15c233de"
|
||||
checksum = "87ed3e93fc7e473e464b9726f4759659e72bc8665e4b8ea227547024f416d905"
|
||||
dependencies = [
|
||||
"const-oid 0.10.2",
|
||||
"crypto-bigint 0.7.0-rc.28",
|
||||
"crypto-bigint 0.7.1",
|
||||
"crypto-primes",
|
||||
"digest 0.11.1",
|
||||
"pkcs1 0.8.0-rc.4",
|
||||
@@ -7260,7 +7267,7 @@ version = "0.0.5"
|
||||
dependencies = [
|
||||
"base64-simd",
|
||||
"rand 0.10.0",
|
||||
"rsa 0.10.0-rc.16",
|
||||
"rsa 0.10.0-rc.17",
|
||||
"serde",
|
||||
"serde_json",
|
||||
]
|
||||
@@ -7675,6 +7682,7 @@ version = "0.0.5"
|
||||
dependencies = [
|
||||
"flate2",
|
||||
"glob",
|
||||
"jiff",
|
||||
"metrics",
|
||||
"nvml-wrapper",
|
||||
"opentelemetry",
|
||||
@@ -7697,7 +7705,6 @@ dependencies = [
|
||||
"tracing-error",
|
||||
"tracing-opentelemetry",
|
||||
"tracing-subscriber",
|
||||
"walkdir",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -8203,7 +8210,7 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f"
|
||||
[[package]]
|
||||
name = "s3s"
|
||||
version = "0.14.0-dev"
|
||||
source = "git+https://github.com/s3s-project/s3s?rev=4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990#4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990"
|
||||
source = "git+https://github.com/s3s-project/s3s?rev=c2dc7b16535659904d4efff52c558fc039be1ef3#c2dc7b16535659904d4efff52c558fc039be1ef3"
|
||||
dependencies = [
|
||||
"arc-swap",
|
||||
"arrayvec",
|
||||
@@ -8271,9 +8278,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "schannel"
|
||||
version = "0.1.28"
|
||||
version = "0.1.29"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1"
|
||||
checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
@@ -8604,9 +8611,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "shadow-rs"
|
||||
version = "1.7.0"
|
||||
version = "1.7.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6d9967e7c3cd89d19cd533d8fceb3e5dcee28cf97fe76441481abe1d32723039"
|
||||
checksum = "3c798acfc78a69c7b038adde44084d8df875555b091da42c90ae46257cdcc41a"
|
||||
dependencies = [
|
||||
"cargo_metadata",
|
||||
"const_format",
|
||||
@@ -9092,9 +9099,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "sysinfo"
|
||||
version = "0.38.3"
|
||||
version = "0.38.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d03c61d2a49c649a15c407338afe7accafde9dac869995dccb73e5f7ef7d9034"
|
||||
checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"memchr",
|
||||
@@ -9143,9 +9150,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "tempfile"
|
||||
version = "3.26.0"
|
||||
version = "3.27.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0"
|
||||
checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd"
|
||||
dependencies = [
|
||||
"fastrand",
|
||||
"getrandom 0.4.2",
|
||||
@@ -10657,18 +10664,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.40"
|
||||
version = "0.8.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a789c6e490b576db9f7e6b6d661bcc9799f7c0ac8352f56ea20193b2681532e5"
|
||||
checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.40"
|
||||
version = "0.8.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f65c489a7071a749c849713807783f70672b28094011623e200cb86dcb835953"
|
||||
checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
24
Cargo.toml
24
Cargo.toml
@@ -144,7 +144,7 @@ flatbuffers = "25.12.19"
|
||||
form_urlencoded = "1.2.2"
|
||||
prost = "0.14.3"
|
||||
quick-xml = "0.39.2"
|
||||
rmcp = { version = "1.1.0" }
|
||||
rmcp = { version = "1.2.0" }
|
||||
rmp = { version = "0.8.15" }
|
||||
rmp-serde = { version = "1.3.1" }
|
||||
serde = { version = "1.0.228", features = ["derive"] }
|
||||
@@ -162,7 +162,7 @@ hmac = { version = "0.13.0-rc.5" }
|
||||
jsonwebtoken = { version = "10.3.0", features = ["aws_lc_rs"] }
|
||||
openidconnect = { version = "4.0", default-features = false }
|
||||
pbkdf2 = "0.13.0-rc.9"
|
||||
rsa = { version = "0.10.0-rc.16" }
|
||||
rsa = { version = "0.10.0-rc.17" }
|
||||
rustls = { version = "0.23.37", default-features = false, features = ["aws-lc-rs", "logging", "tls12", "prefer-post-quantum", "std"] }
|
||||
rustls-pki-types = "1.14.0"
|
||||
sha1 = "0.11.0-rc.5"
|
||||
@@ -184,7 +184,7 @@ atoi = "2.0.0"
|
||||
atomic_enum = "0.3.0"
|
||||
aws-config = { version = "1.8.15" }
|
||||
aws-credential-types = { version = "1.2.14" }
|
||||
aws-sdk-s3 = { version = "1.125.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-sdk-s3 = { version = "1.126.0", default-features = false, features = ["sigv4a", "default-https-client", "rt-tokio"] }
|
||||
aws-smithy-http-client = { version = "1.1.12", default-features = false, features = ["default-client", "rustls-aws-lc"] }
|
||||
aws-smithy-types = { version = "1.4.6" }
|
||||
backtrace = "0.3.76"
|
||||
@@ -192,19 +192,19 @@ base64 = "0.22.1"
|
||||
base64-simd = "0.8.0"
|
||||
brotli = "8.0.2"
|
||||
cfg-if = "1.0.4"
|
||||
clap = { version = "4.5.60", features = ["derive", "env"] }
|
||||
clap = { version = "4.6.0", features = ["derive", "env"] }
|
||||
const-str = { version = "1.1.0", features = ["std", "proc"] }
|
||||
convert_case = "0.11.0"
|
||||
criterion = { version = "0.8", features = ["html_reports"] }
|
||||
crossbeam-queue = "0.3.12"
|
||||
datafusion = "52.2.0"
|
||||
datafusion = "52.3.0"
|
||||
derive_builder = "0.20.2"
|
||||
enumset = "1.1.10"
|
||||
faster-hex = "0.10.0"
|
||||
flate2 = "1.1.9"
|
||||
glob = "0.3.3"
|
||||
google-cloud-storage = "1.8.0"
|
||||
google-cloud-auth = "1.6.0"
|
||||
google-cloud-storage = "1.9.0"
|
||||
google-cloud-auth = "1.7.0"
|
||||
hashbrown = { version = "0.16.1", features = ["serde", "rayon"] }
|
||||
hex = "0.4.3"
|
||||
hex-simd = "0.8.0"
|
||||
@@ -239,9 +239,9 @@ rumqttc = { version = "0.25.1" }
|
||||
rustix = { version = "1.1.4", features = ["fs"] }
|
||||
rust-embed = { version = "8.11.0" }
|
||||
rustc-hash = { version = "2.1.1" }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "4189c86154ed1fa4b89d2eaf2acfe04b7dcd0990", features = ["minio"] }
|
||||
s3s = { git = "https://github.com/s3s-project/s3s", rev = "c2dc7b16535659904d4efff52c558fc039be1ef3", features = ["minio"] }
|
||||
serial_test = "3.4.0"
|
||||
shadow-rs = { version = "1.7.0", default-features = false }
|
||||
shadow-rs = { version = "1.7.1", default-features = false }
|
||||
siphasher = "1.0.2"
|
||||
smallvec = { version = "1.15.1", features = ["serde"] }
|
||||
smartstring = "1.0.1"
|
||||
@@ -249,9 +249,9 @@ snafu = "0.9.0"
|
||||
snap = "1.1.1"
|
||||
starshard = { version = "1.1.0", features = ["rayon", "async", "serde"] }
|
||||
strum = { version = "0.28.0", features = ["derive"] }
|
||||
sysinfo = "0.38.3"
|
||||
sysinfo = "0.38.4"
|
||||
temp-env = "0.3.6"
|
||||
tempfile = "3.26.0"
|
||||
tempfile = "3.27.0"
|
||||
test-case = "3.3.1"
|
||||
thiserror = "2.0.18"
|
||||
tracing = { version = "0.1.44" }
|
||||
@@ -279,7 +279,7 @@ opentelemetry-otlp = { version = "0.31.0", features = ["gzip-http", "reqwest-rus
|
||||
opentelemetry_sdk = { version = "0.31.0" }
|
||||
opentelemetry-semantic-conventions = { version = "0.31.0", features = ["semconv_experimental"] }
|
||||
opentelemetry-stdout = { version = "0.31.0" }
|
||||
pyroscope = { version = "1.0.2", features = ["backend-pprof-rs"] }
|
||||
pyroscope = { version = "2.0.0", features = ["backend-pprof-rs"] }
|
||||
|
||||
# FTP and SFTP
|
||||
libunftp = { version = "0.23.0", features = ["experimental"] }
|
||||
|
||||
@@ -38,6 +38,7 @@ rustfs-config = { workspace = true, features = ["constants", "observability"] }
|
||||
rustfs-utils = { workspace = true, features = ["ip", "path"] }
|
||||
flate2 = { workspace = true }
|
||||
glob = { workspace = true }
|
||||
jiff = { workspace = true }
|
||||
metrics = { workspace = true }
|
||||
nvml-wrapper = { workspace = true, optional = true }
|
||||
opentelemetry = { workspace = true }
|
||||
@@ -56,7 +57,6 @@ tracing-subscriber = { workspace = true, features = ["registry", "std", "fmt", "
|
||||
tokio = { workspace = true, features = ["sync", "fs", "rt-multi-thread", "rt", "time", "macros"] }
|
||||
sysinfo = { workspace = true }
|
||||
thiserror = { workspace = true }
|
||||
walkdir = { workspace = true }
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
pyroscope = { workspace = true, features = ["backend-pprof-rs"] }
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# rustfs-obs
|
||||
|
||||
Observability library for [RustFS](https://github.com/rustfs/rustfs) providing structured JSON
|
||||
logging, distributed tracing, and metrics via OpenTelemetry.
|
||||
logging, distributed tracing, metrics via OpenTelemetry, and continuous profiling via Pyroscope.
|
||||
|
||||
---
|
||||
|
||||
@@ -10,9 +10,10 @@ logging, distributed tracing, and metrics via OpenTelemetry.
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| **Structured logging** | JSON-formatted logs via `tracing-subscriber` |
|
||||
| **Rolling-file logging** | Daily / hourly rotation with automatic cleanup |
|
||||
| **Rolling-file logging** | Daily / hourly rotation with automatic cleanup and high-precision timestamps |
|
||||
| **Distributed tracing** | OTLP/HTTP export to Jaeger, Tempo, or any OTel collector |
|
||||
| **Metrics** | OTLP/HTTP export, bridged from the `metrics` crate facade |
|
||||
| **Continuous Profiling** | CPU/Memory profiling export to Pyroscope |
|
||||
| **Log cleanup** | Background task: size limits, gzip compression, retention policies |
|
||||
| **GPU metrics** *(optional)* | Enable with the `gpu` feature flag |
|
||||
|
||||
@@ -79,7 +80,7 @@ The library selects a backend automatically based on configuration:
|
||||
|
||||
```
|
||||
1. Any OTLP endpoint set?
|
||||
└─ YES → Full OTLP/HTTP pipeline (traces + metrics + logs)
|
||||
└─ YES → Full OTLP/HTTP pipeline (traces + metrics + logs + profiling)
|
||||
|
||||
2. RUSTFS_OBS_LOG_DIRECTORY set to a non-empty path?
|
||||
└─ YES → Rolling-file JSON logging
|
||||
@@ -110,9 +111,11 @@ All configuration is read from environment variables at startup.
|
||||
| `RUSTFS_OBS_TRACE_ENDPOINT` | _(empty)_ | Dedicated trace endpoint (overrides root + `/v1/traces`) |
|
||||
| `RUSTFS_OBS_METRIC_ENDPOINT` | _(empty)_ | Dedicated metrics endpoint |
|
||||
| `RUSTFS_OBS_LOG_ENDPOINT` | _(empty)_ | Dedicated log endpoint |
|
||||
| `RUSTFS_OBS_PROFILING_ENDPOINT` | _(empty)_ | Dedicated profiling endpoint (e.g. Pyroscope) |
|
||||
| `RUSTFS_OBS_TRACES_EXPORT_ENABLED` | `true` | Toggle trace export |
|
||||
| `RUSTFS_OBS_METRICS_EXPORT_ENABLED` | `true` | Toggle metrics export |
|
||||
| `RUSTFS_OBS_LOGS_EXPORT_ENABLED` | `true` | Toggle OTLP log export |
|
||||
| `RUSTFS_OBS_PROFILING_EXPORT_ENABLED` | `true` | Toggle profiling export |
|
||||
| `RUSTFS_OBS_USE_STDOUT` | `false` | Mirror all signals to stdout alongside OTLP |
|
||||
| `RUSTFS_OBS_SAMPLE_RATIO` | `0.1` | Trace sampling ratio `0.0`–`1.0` |
|
||||
| `RUSTFS_OBS_METER_INTERVAL` | `15` | Metrics export interval (seconds) |
|
||||
@@ -132,7 +135,7 @@ All configuration is read from environment variables at startup.
|
||||
| `RUSTFS_OBS_LOGGER_LEVEL` | `info` | Log level; `RUST_LOG` syntax supported |
|
||||
| `RUSTFS_OBS_LOG_STDOUT_ENABLED` | `false` | When file logging is active, also mirror to stdout |
|
||||
| `RUSTFS_OBS_LOG_DIRECTORY` | _(empty)_ | **Directory for rolling log files. When empty, logs go to stdout only** |
|
||||
| `RUSTFS_OBS_LOG_FILENAME` | `rustfs.log` | Base filename for rolling logs (date suffix added automatically) |
|
||||
| `RUSTFS_OBS_LOG_FILENAME` | `rustfs.log` | Base filename for rolling logs. Rotated archives include a high-precision timestamp and counter. With the default `RUSTFS_OBS_LOG_MATCH_MODE=suffix`, names look like `<timestamp>-<counter>.rustfs.log` (e.g., `20231027103001.123456-0.rustfs.log`); with `prefix`, they look like `rustfs.log.<timestamp>-<counter>` (e.g., `rustfs.log.20231027103001.123456-0`). |
|
||||
| `RUSTFS_OBS_LOG_ROTATION_TIME` | `hourly` | Rotation granularity: `minutely`, `hourly`, or `daily` |
|
||||
| `RUSTFS_OBS_LOG_KEEP_FILES` | `30` | Number of rolling files to keep (also used by cleaner) |
|
||||
| `RUSTFS_OBS_LOG_MATCH_MODE` | `suffix` | File matching mode: `prefix` or `suffix` |
|
||||
@@ -247,21 +250,23 @@ use std::path::PathBuf;
|
||||
use rustfs_obs::LogCleaner;
|
||||
use rustfs_obs::types::FileMatchMode;
|
||||
|
||||
let cleaner = LogCleaner::new(
|
||||
let cleaner = LogCleaner::builder(
|
||||
PathBuf::from("/var/log/rustfs"),
|
||||
"rustfs.log.".to_string(), // file_pattern
|
||||
FileMatchMode::Prefix, // match_mode
|
||||
10, // keep_count
|
||||
2 * 1024 * 1024 * 1024, // max_total_size_bytes (2 GiB)
|
||||
0, // max_single_file_size_bytes (unlimited)
|
||||
true, // compress_old_files
|
||||
6, // gzip_compression_level
|
||||
30, // compressed_file_retention_days
|
||||
vec!["current.log".to_string()], // exclude_patterns
|
||||
true, // delete_empty_files
|
||||
3600, // min_file_age_seconds (1 hour)
|
||||
false, // dry_run
|
||||
);
|
||||
"rustfs.log".to_string(), // active_filename
|
||||
)
|
||||
.match_mode(FileMatchMode::Prefix)
|
||||
.keep_files(10)
|
||||
.max_total_size_bytes(2 * 1024 * 1024 * 1024) // 2 GiB
|
||||
.max_single_file_size_bytes(0) // unlimited
|
||||
.compress_old_files(true)
|
||||
.gzip_compression_level(6)
|
||||
.compressed_file_retention_days(30)
|
||||
.exclude_patterns(vec!["current.log".to_string()])
|
||||
.delete_empty_files(true)
|
||||
.min_file_age_seconds(3600) // 1 hour
|
||||
.dry_run(false)
|
||||
.build();
|
||||
|
||||
let (deleted, freed_bytes) = cleaner.cleanup().expect("cleanup failed");
|
||||
println!("Deleted {deleted} files, freed {freed_bytes} bytes");
|
||||
|
||||
71
crates/obs/src/cleaner/README.md
Normal file
71
crates/obs/src/cleaner/README.md
Normal file
@@ -0,0 +1,71 @@
|
||||
# Log Cleaner Subsystem
|
||||
|
||||
The `cleaner` module provides a robust, background log-file lifecycle manager for RustFS. It is designed to run periodically to enforce retention policies, compress old logs, and prevent disk exhaustion.
|
||||
|
||||
## Architecture
|
||||
|
||||
The cleaner operates as a pipeline:
|
||||
|
||||
1. **Discovery (`scanner.rs`)**: Scans the configured log directory for eligible files.
|
||||
* **Non-recursive**: Only scans the top-level directory for safety.
|
||||
* **Filtering**: Ignores the currently active log file, files matching exclude patterns, and files that do not match the configured prefix/suffix pattern.
|
||||
* **Performance**: Uses `std::fs::read_dir` directly to minimize overhead and syscalls.
|
||||
|
||||
2. **Selection (`core.rs`)**: Applies retention policies to select files for deletion.
|
||||
* **Keep Count**: Ensures at least `N` recent files are kept.
|
||||
* **Total Size**: Deletes oldest files if the total size exceeds the limit.
|
||||
* **Single File Size**: Deletes individual files that exceed a size limit (e.g., runaway logs).
|
||||
|
||||
3. **Action (`core.rs` / `compress.rs`)**:
|
||||
* **Compression**: Optionally compresses selected files using Gzip (level 1-9) before deletion.
|
||||
* **Deletion**: Removes the original file (and eventually the compressed archive based on retention days).
|
||||
|
||||
## Configuration
|
||||
|
||||
The cleaner is configured via `LogCleanerBuilder`. When initialized via `rustfs-obs::init_obs`, it reads from environment variables.
|
||||
|
||||
| Parameter | Env Var | Description |
|
||||
|-----------|---------|-------------|
|
||||
| `log_dir` | `RUSTFS_OBS_LOG_DIRECTORY` | The directory to scan. |
|
||||
| `file_pattern` | `RUSTFS_OBS_LOG_FILENAME` | The base filename pattern (e.g., `rustfs.log`). |
|
||||
| `active_filename` | (Derived) | The exact name of the currently active log file, excluded from cleanup. |
|
||||
| `match_mode` | `RUSTFS_OBS_LOG_MATCH_MODE` | `prefix` or `suffix`. Determines how `file_pattern` is matched against filenames. |
|
||||
| `keep_files` | `RUSTFS_OBS_LOG_KEEP_FILES` | Minimum number of rolling log files to keep. |
|
||||
| `max_total_size_bytes` | `RUSTFS_OBS_LOG_MAX_TOTAL_SIZE_BYTES` | Maximum aggregate size of all log files. Oldest files are deleted to satisfy this. |
|
||||
| `compress_old_files` | `RUSTFS_OBS_LOG_COMPRESS_OLD_FILES` | If `true`, files selected for removal are first gzipped. |
|
||||
| `compressed_file_retention_days` | `RUSTFS_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS` | Age in days after which `.gz` files are deleted. |
|
||||
|
||||
## Timestamp Format & Rotation
|
||||
|
||||
The cleaner works in tandem with the `RollingAppender` in `telemetry/rolling.rs`.
|
||||
|
||||
* **Rotation**: Logs are rotated based on time (Daily/Hourly/Minutely) or Size.
|
||||
* **Naming**: Archived logs use a high-precision timestamp format: `YYYYMMDDHHMMSS.uuuuuu` (microseconds), plus a unique counter to prevent collisions.
|
||||
* **Suffix Mode**: `<timestamp>-<counter>.<filename>` (e.g., `20231027103001.123456-0.rustfs.log`)
|
||||
* **Prefix Mode**: `<filename>.<timestamp>-<counter>` (e.g., `rustfs.log.20231027103001.123456-0`)
|
||||
|
||||
This high-precision naming ensures that files sort chronologically by name, and collisions are virtually impossible even under high load.
|
||||
|
||||
## Usage Example
|
||||
|
||||
```rust
|
||||
use rustfs_obs::LogCleaner;
|
||||
use rustfs_obs::types::FileMatchMode;
|
||||
use std::path::PathBuf;
|
||||
|
||||
let cleaner = LogCleaner::builder(
|
||||
PathBuf::from("/var/log/rustfs"),
|
||||
"rustfs.log.".to_string(),
|
||||
"rustfs.log".to_string(),
|
||||
)
|
||||
.match_mode(FileMatchMode::Prefix)
|
||||
.keep_files(10)
|
||||
.max_total_size_bytes(1024 * 1024 * 100) // 100 MB
|
||||
.compress_old_files(true)
|
||||
.build();
|
||||
|
||||
// Run cleanup (blocking operation, spawn in a background task)
|
||||
if let Ok((deleted, freed)) = cleaner.cleanup() {
|
||||
println!("Cleaned up {} files, freed {} bytes", deleted, freed);
|
||||
}
|
||||
```
|
||||
@@ -15,7 +15,7 @@
|
||||
//! Core log-file cleanup orchestration.
|
||||
//!
|
||||
//! [`LogCleaner`] is the public entry point for the cleanup subsystem.
|
||||
//! Construct it with [`LogCleaner::new`] and call [`LogCleaner::cleanup`]
|
||||
//! Construct it with [`LogCleaner::builder`] and call [`LogCleaner::cleanup`]
|
||||
//! periodically (e.g. from a `tokio::spawn`-ed loop).
|
||||
//!
|
||||
//! Internally the cleaner delegates to:
|
||||
@@ -24,9 +24,11 @@
|
||||
//! - [`LogCleaner::select_files_to_delete`] — to apply count / size limits.
|
||||
|
||||
use super::compress::compress_file;
|
||||
use super::scanner::{collect_expired_compressed_files, collect_log_files};
|
||||
use super::scanner::{LogScanResult, scan_log_directory};
|
||||
use super::types::{FileInfo, FileMatchMode};
|
||||
use rustfs_config::DEFAULT_LOG_KEEP_FILES;
|
||||
use std::path::PathBuf;
|
||||
use std::time::SystemTime;
|
||||
use tracing::{debug, error, info};
|
||||
|
||||
/// Log-file lifecycle manager.
|
||||
@@ -43,6 +45,8 @@ pub struct LogCleaner {
|
||||
pub(super) log_dir: PathBuf,
|
||||
/// Pattern string to match files (used as prefix or suffix).
|
||||
pub(super) file_pattern: String,
|
||||
/// Exact name of the active log file (to exclude from cleanup).
|
||||
pub(super) active_filename: String,
|
||||
/// Whether to match by prefix or suffix.
|
||||
pub(super) match_mode: FileMatchMode,
|
||||
/// The cleaner will never delete files if doing so would leave fewer than
|
||||
@@ -70,54 +74,19 @@ pub struct LogCleaner {
|
||||
}
|
||||
|
||||
impl LogCleaner {
|
||||
/// Build a new [`LogCleaner`] with the supplied policy parameters.
|
||||
///
|
||||
/// `exclude_patterns` is a list of glob strings (e.g. `"*.lock"`). Invalid
|
||||
/// glob patterns are silently ignored.
|
||||
///
|
||||
/// `gzip_compression_level` is clamped to the range `[1, 9]`.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn new(
|
||||
log_dir: PathBuf,
|
||||
file_pattern: String,
|
||||
match_mode: FileMatchMode,
|
||||
keep_files: usize,
|
||||
max_total_size_bytes: u64,
|
||||
max_single_file_size_bytes: u64,
|
||||
compress_old_files: bool,
|
||||
gzip_compression_level: u32,
|
||||
compressed_file_retention_days: u64,
|
||||
exclude_patterns: Vec<String>,
|
||||
delete_empty_files: bool,
|
||||
min_file_age_seconds: u64,
|
||||
dry_run: bool,
|
||||
) -> Self {
|
||||
let patterns = exclude_patterns
|
||||
.into_iter()
|
||||
.filter_map(|p| glob::Pattern::new(&p).ok())
|
||||
.collect();
|
||||
|
||||
Self {
|
||||
log_dir,
|
||||
file_pattern,
|
||||
match_mode,
|
||||
keep_files,
|
||||
max_total_size_bytes,
|
||||
max_single_file_size_bytes,
|
||||
compress_old_files,
|
||||
gzip_compression_level: gzip_compression_level.clamp(1, 9),
|
||||
compressed_file_retention_days,
|
||||
exclude_patterns: patterns,
|
||||
delete_empty_files,
|
||||
min_file_age_seconds,
|
||||
dry_run,
|
||||
}
|
||||
/// Create a builder to construct a `LogCleaner`.
|
||||
pub fn builder(
|
||||
log_dir: impl Into<PathBuf>,
|
||||
file_pattern: impl Into<String>,
|
||||
active_filename: impl Into<String>,
|
||||
) -> LogCleanerBuilder {
|
||||
LogCleanerBuilder::new(log_dir, file_pattern, active_filename)
|
||||
}
|
||||
|
||||
/// Perform one full cleanup pass.
|
||||
///
|
||||
/// Steps:
|
||||
/// 1. Scan the log directory for managed files.
|
||||
/// 1. Scan the log directory for managed files (excluding the active file).
|
||||
/// 2. Apply count/size policies to select files for deletion.
|
||||
/// 3. Optionally compress selected files, then delete them.
|
||||
/// 4. Collect and delete expired compressed archives.
|
||||
@@ -137,10 +106,15 @@ impl LogCleaner {
|
||||
let mut total_deleted = 0usize;
|
||||
let mut total_freed = 0u64;
|
||||
|
||||
// ── 1. Discover active log files ──────────────────────────────────────
|
||||
let mut files = collect_log_files(
|
||||
// ── 1. Discover active log files (Archives only) ──────────────────────
|
||||
// We explicitly pass `active_filename` to exclude it from the list.
|
||||
let LogScanResult {
|
||||
mut logs,
|
||||
mut compressed_archives,
|
||||
} = scan_log_directory(
|
||||
&self.log_dir,
|
||||
&self.file_pattern,
|
||||
Some(&self.active_filename),
|
||||
self.match_mode,
|
||||
&self.exclude_patterns,
|
||||
self.min_file_age_seconds,
|
||||
@@ -148,27 +122,19 @@ impl LogCleaner {
|
||||
self.dry_run,
|
||||
)?;
|
||||
|
||||
if files.is_empty() {
|
||||
debug!("No log files found in directory: {:?}", self.log_dir);
|
||||
} else {
|
||||
files.sort_by_key(|f| f.modified);
|
||||
let total_size: u64 = files.iter().map(|f| f.size).sum();
|
||||
// ── 2. Select + compress + delete (Regular Logs) ──────────────────────
|
||||
if !logs.is_empty() {
|
||||
logs.sort_by_key(|f| f.modified);
|
||||
let total_size: u64 = logs.iter().map(|f| f.size).sum();
|
||||
|
||||
info!(
|
||||
"Found {} log files, total size: {} bytes ({:.2} MB)",
|
||||
files.len(),
|
||||
"Found {} regular log files, total size: {} bytes ({:.2} MB)",
|
||||
logs.len(),
|
||||
total_size,
|
||||
total_size as f64 / 1024.0 / 1024.0
|
||||
);
|
||||
|
||||
// ── 2. Select + compress + delete ─────────────────────────────────
|
||||
let (to_delete, to_rotate) = self.select_files_to_process(&files, total_size);
|
||||
|
||||
// Handle rotation for active file if needed
|
||||
if let Some(active_file) = to_rotate
|
||||
&& let Err(e) = self.rotate_active_file(&active_file)
|
||||
{
|
||||
error!("Failed to rotate active file {:?}: {}", active_file.path, e);
|
||||
}
|
||||
let to_delete = self.select_files_to_process(&logs, total_size);
|
||||
|
||||
if !to_delete.is_empty() {
|
||||
let (d, f) = self.compress_and_delete(&to_delete)?;
|
||||
@@ -178,16 +144,13 @@ impl LogCleaner {
|
||||
}
|
||||
|
||||
// ── 3. Remove expired compressed archives ─────────────────────────────
|
||||
let expired_gz = collect_expired_compressed_files(
|
||||
&self.log_dir,
|
||||
&self.file_pattern,
|
||||
self.match_mode,
|
||||
self.compressed_file_retention_days,
|
||||
)?;
|
||||
if !expired_gz.is_empty() {
|
||||
let (d, f) = self.delete_files(&expired_gz)?;
|
||||
total_deleted += d;
|
||||
total_freed += f;
|
||||
if !compressed_archives.is_empty() && self.compressed_file_retention_days > 0 {
|
||||
let expired = self.select_expired_compressed(&mut compressed_archives);
|
||||
if !expired.is_empty() {
|
||||
let (d, f) = self.delete_files(&expired)?;
|
||||
total_deleted += d;
|
||||
total_freed += f;
|
||||
}
|
||||
}
|
||||
|
||||
if total_deleted > 0 || total_freed > 0 {
|
||||
@@ -204,28 +167,19 @@ impl LogCleaner {
|
||||
|
||||
// ─── Selection ────────────────────────────────────────────────────────────
|
||||
|
||||
/// Choose which files from `files` (sorted oldest-first) should be deleted or rotated.
|
||||
/// Choose which files from `files` (sorted oldest-first) should be deleted.
|
||||
///
|
||||
/// The algorithm respects three constraints in order:
|
||||
/// 1. Always keep at least `keep_files` files.
|
||||
/// 1. Always keep at least `keep_files` files (archives).
|
||||
/// 2. Delete old files while the total size exceeds `max_total_size_bytes`.
|
||||
/// 3. Delete any file whose individual size exceeds `max_single_file_size_bytes`.
|
||||
///
|
||||
/// **Note**: The most recent file (assumed to be the active log) is exempt
|
||||
/// from size-based deletion. If it exceeds the size limit, it is returned
|
||||
/// as `to_rotate`.
|
||||
pub(super) fn select_files_to_process(&self, files: &[FileInfo], total_size: u64) -> (Vec<FileInfo>, Option<FileInfo>) {
|
||||
pub(super) fn select_files_to_process(&self, files: &[FileInfo], total_size: u64) -> Vec<FileInfo> {
|
||||
let mut to_delete = Vec::new();
|
||||
let mut to_rotate = None;
|
||||
|
||||
if files.is_empty() {
|
||||
return (to_delete, to_rotate);
|
||||
return to_delete;
|
||||
}
|
||||
|
||||
// Identify the index of the most recent file (last in the sorted list).
|
||||
// We will protect this file from size-based deletion.
|
||||
let active_file_idx = files.len() - 1;
|
||||
|
||||
// Calculate how many files we *must* delete to satisfy keep_files.
|
||||
let must_delete_count = files.len().saturating_sub(self.keep_files);
|
||||
|
||||
@@ -244,142 +198,111 @@ impl LogCleaner {
|
||||
let over_total = self.max_total_size_bytes > 0 && current_size > self.max_total_size_bytes;
|
||||
|
||||
// Condition 3: Enforce max_single_file_size_bytes.
|
||||
// Note: Since active file is excluded, if an archive is > max_single, it means it
|
||||
// was rotated out being too large (likely) or we lowered the limit. It should be deleted.
|
||||
let over_single = self.max_single_file_size_bytes > 0 && file.size > self.max_single_file_size_bytes;
|
||||
|
||||
if over_total {
|
||||
// If we are over total size, we delete unless it's the active file.
|
||||
if idx == active_file_idx {
|
||||
debug!(
|
||||
"Active log file contributes to total size limit overflow, but skipping deletion to preserve current logs."
|
||||
);
|
||||
} else {
|
||||
current_size = current_size.saturating_sub(file.size);
|
||||
to_delete.push(file.clone());
|
||||
}
|
||||
current_size = current_size.saturating_sub(file.size);
|
||||
to_delete.push(file.clone());
|
||||
} else if over_single {
|
||||
// For single file limits, we MUST NOT delete the active file.
|
||||
if idx == active_file_idx {
|
||||
// Mark active file for rotation instead of deletion
|
||||
to_rotate = Some(file.clone());
|
||||
} else {
|
||||
debug!(
|
||||
"File exceeds single-file size limit: {:?} ({} > {} bytes)",
|
||||
file.path, file.size, self.max_single_file_size_bytes
|
||||
);
|
||||
current_size = current_size.saturating_sub(file.size);
|
||||
to_delete.push(file.clone());
|
||||
}
|
||||
debug!(
|
||||
"Archive exceeds single-file size limit: {:?} ({} > {} bytes). Deleting.",
|
||||
file.path, file.size, self.max_single_file_size_bytes
|
||||
);
|
||||
current_size = current_size.saturating_sub(file.size);
|
||||
to_delete.push(file.clone());
|
||||
}
|
||||
}
|
||||
|
||||
(to_delete, to_rotate)
|
||||
to_delete
|
||||
}
|
||||
|
||||
// ─── Rotation ─────────────────────────────────────────────────────────────
|
||||
/// Select compressed files that have exceeded the retention period.
|
||||
fn select_expired_compressed(&self, files: &mut [FileInfo]) -> Vec<FileInfo> {
|
||||
let retention = std::time::Duration::from_secs(self.compressed_file_retention_days * 24 * 3600);
|
||||
let now = SystemTime::now();
|
||||
let mut expired = Vec::new();
|
||||
|
||||
/// Rotate the active file by renaming it with a timestamp suffix.
|
||||
/// The original filename will be recreated by the logging appender on next write.
|
||||
fn rotate_active_file(&self, file: &FileInfo) -> Result<(), std::io::Error> {
|
||||
if self.dry_run {
|
||||
info!("[DRY RUN] Would rotate active file: {:?} ({} bytes)", file.path, file.size);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Generate timestamp: unix timestamp in seconds
|
||||
let timestamp = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.map_err(std::io::Error::other)?
|
||||
.as_secs();
|
||||
|
||||
let file_name = file
|
||||
.path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.ok_or_else(|| std::io::Error::new(std::io::ErrorKind::InvalidData, "Invalid filename"))?;
|
||||
|
||||
// Construct the rotated filename.
|
||||
// We must ensure the new filename still matches the file_pattern so it can be discovered
|
||||
// by the scanner in future runs (and eventually deleted).
|
||||
//
|
||||
// Suffix mode: Insert timestamp BEFORE the suffix.
|
||||
// Example: "2026-03-01.rustfs.log" (pattern="rustfs.log")
|
||||
// -> "2026-03-01.1740810000.rustfs.log"
|
||||
//
|
||||
// Prefix mode: Append timestamp at the end.
|
||||
// Example: "app.log" (pattern="app")
|
||||
// -> "app.log.1740810000"
|
||||
let rotated_name = match self.match_mode {
|
||||
FileMatchMode::Suffix => {
|
||||
if let Some(base) = file_name.strip_suffix(&self.file_pattern) {
|
||||
let mut new_name = String::with_capacity(file_name.len() + 20);
|
||||
new_name.push_str(base);
|
||||
|
||||
// Ensure separator between base and timestamp
|
||||
if !base.is_empty() && !base.ends_with('.') {
|
||||
new_name.push('.');
|
||||
}
|
||||
new_name.push_str(×tamp.to_string());
|
||||
|
||||
// Ensure separator between timestamp and suffix
|
||||
if !self.file_pattern.starts_with('.') {
|
||||
new_name.push('.');
|
||||
}
|
||||
new_name.push_str(&self.file_pattern);
|
||||
|
||||
new_name
|
||||
} else {
|
||||
// Should not happen if scanner works correctly, but fallback safely
|
||||
format!("{}.{}", file_name, timestamp)
|
||||
}
|
||||
for file in files {
|
||||
if let Ok(age) = now.duration_since(file.modified)
|
||||
&& age > retention
|
||||
{
|
||||
expired.push(file.clone());
|
||||
}
|
||||
FileMatchMode::Prefix => {
|
||||
// For prefix matching, appending to the end preserves the prefix.
|
||||
format!("{}.{}", file_name, timestamp)
|
||||
}
|
||||
};
|
||||
|
||||
let rotated_path = file.path.with_file_name(&rotated_name);
|
||||
|
||||
// Check if target already exists to avoid overwriting (unlikely with timestamp but possible)
|
||||
if rotated_path.exists() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::AlreadyExists,
|
||||
format!("Rotated file already exists: {:?}", rotated_path),
|
||||
));
|
||||
}
|
||||
|
||||
info!("Rotating active log file: {:?} -> {:?}", file.path, rotated_path);
|
||||
|
||||
// Rename the current active file to the rotated name.
|
||||
// The logging appender (tracing-appender) will automatically create a new file
|
||||
// with the original name when it next attempts to write.
|
||||
// Note: On Linux/Unix, this rename is atomic and safe even if the file is open.
|
||||
if let Err(e) = std::fs::rename(&file.path, &rotated_path) {
|
||||
// Add context to the error
|
||||
return Err(std::io::Error::new(
|
||||
e.kind(),
|
||||
format!("Failed to rename {:?} to {:?}: {}", file.path, rotated_path, e),
|
||||
));
|
||||
}
|
||||
|
||||
Ok(())
|
||||
expired
|
||||
}
|
||||
|
||||
// ─── Compression + deletion ───────────────────────────────────────────────
|
||||
|
||||
/// Securely delete a file, preventing symlink attacks (TOCTOU).
|
||||
///
|
||||
/// This function verifies that the path is not a symlink before attempting deletion.
|
||||
/// While strictly speaking a race condition is still theoretically possible between
|
||||
/// `symlink_metadata` and `remove_file`, this check covers the vast majority of
|
||||
/// privilege escalation vectors where a user replaces a log file with a symlink
|
||||
/// to a system file.
|
||||
fn secure_delete(&self, path: &PathBuf) -> std::io::Result<()> {
|
||||
// 1. Lstat (symlink_metadata) - do not follow links
|
||||
let meta = std::fs::symlink_metadata(path)?;
|
||||
|
||||
// 2. Symlink Check
|
||||
// If it's a symlink, we NEVER delete it. It might point to /etc/passwd.
|
||||
// In a log directory, symlinks are unexpected and dangerous.
|
||||
if meta.file_type().is_symlink() {
|
||||
return Err(std::io::Error::new(
|
||||
std::io::ErrorKind::InvalidData,
|
||||
format!("Security: refusing to delete symlink: {:?}", path),
|
||||
));
|
||||
}
|
||||
|
||||
// 3. Perform Deletion
|
||||
std::fs::remove_file(path)
|
||||
}
|
||||
|
||||
/// Optionally compress and then delete the given files.
|
||||
///
|
||||
/// This function is synchronous and blocking. It should be called within a
|
||||
/// `spawn_blocking` task if running in an async context.
|
||||
fn compress_and_delete(&self, files: &[FileInfo]) -> Result<(usize, u64), std::io::Error> {
|
||||
if self.compress_old_files {
|
||||
for f in files {
|
||||
if let Err(e) = compress_file(&f.path, self.gzip_compression_level, self.dry_run) {
|
||||
tracing::warn!("Failed to compress {:?}: {}", f.path, e);
|
||||
let mut total_deleted = 0;
|
||||
let mut total_freed = 0;
|
||||
|
||||
for f in files {
|
||||
let mut deleted_size = 0;
|
||||
if self.compress_old_files {
|
||||
match compress_file(&f.path, self.gzip_compression_level, self.dry_run) {
|
||||
Ok(_) => {}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to compress {:?}: {}", f.path, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Now delete
|
||||
if self.dry_run {
|
||||
info!("[DRY RUN] Would delete: {:?} ({} bytes)", f.path, f.size);
|
||||
deleted_size = f.size;
|
||||
} else {
|
||||
match self.secure_delete(&f.path) {
|
||||
Ok(()) => {
|
||||
debug!("Deleted: {:?}", f.path);
|
||||
deleted_size = f.size;
|
||||
}
|
||||
Err(e) => {
|
||||
error!("Failed to delete {:?}: {}", f.path, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
if deleted_size > 0 {
|
||||
total_deleted += 1;
|
||||
total_freed += deleted_size;
|
||||
}
|
||||
}
|
||||
self.delete_files(files)
|
||||
|
||||
Ok((total_deleted, total_freed))
|
||||
}
|
||||
|
||||
/// Delete all files in `files`, logging each operation.
|
||||
@@ -398,7 +321,7 @@ impl LogCleaner {
|
||||
deleted += 1;
|
||||
freed += f.size;
|
||||
} else {
|
||||
match std::fs::remove_file(&f.path) {
|
||||
match self.secure_delete(&f.path) {
|
||||
Ok(()) => {
|
||||
debug!("Deleted: {:?}", f.path);
|
||||
deleted += 1;
|
||||
@@ -414,3 +337,125 @@ impl LogCleaner {
|
||||
Ok((deleted, freed))
|
||||
}
|
||||
}
|
||||
|
||||
/// Builder for [`LogCleaner`].
|
||||
pub struct LogCleanerBuilder {
|
||||
log_dir: PathBuf,
|
||||
file_pattern: String,
|
||||
active_filename: String,
|
||||
match_mode: FileMatchMode,
|
||||
keep_files: usize,
|
||||
max_total_size_bytes: u64,
|
||||
max_single_file_size_bytes: u64,
|
||||
compress_old_files: bool,
|
||||
gzip_compression_level: u32,
|
||||
compressed_file_retention_days: u64,
|
||||
exclude_patterns: Vec<String>,
|
||||
delete_empty_files: bool,
|
||||
min_file_age_seconds: u64,
|
||||
dry_run: bool,
|
||||
}
|
||||
|
||||
impl LogCleanerBuilder {
|
||||
pub fn new(log_dir: impl Into<PathBuf>, file_pattern: impl Into<String>, active_filename: impl Into<String>) -> Self {
|
||||
Self {
|
||||
log_dir: log_dir.into(),
|
||||
file_pattern: file_pattern.into(),
|
||||
active_filename: active_filename.into(),
|
||||
match_mode: FileMatchMode::Prefix,
|
||||
// Default to a safe non-zero value so that a builder created
|
||||
// without an explicit `keep_files()` call does not immediately
|
||||
// delete all matching log files.
|
||||
keep_files: DEFAULT_LOG_KEEP_FILES,
|
||||
max_total_size_bytes: 0,
|
||||
max_single_file_size_bytes: 0,
|
||||
compress_old_files: false,
|
||||
gzip_compression_level: 6,
|
||||
compressed_file_retention_days: 0,
|
||||
exclude_patterns: Vec::new(),
|
||||
delete_empty_files: false,
|
||||
min_file_age_seconds: 0,
|
||||
dry_run: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn match_mode(mut self, match_mode: FileMatchMode) -> Self {
|
||||
self.match_mode = match_mode;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn keep_files(mut self, keep_files: usize) -> Self {
|
||||
self.keep_files = keep_files;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn max_total_size_bytes(mut self, max_total_size_bytes: u64) -> Self {
|
||||
self.max_total_size_bytes = max_total_size_bytes;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn max_single_file_size_bytes(mut self, max_single_file_size_bytes: u64) -> Self {
|
||||
self.max_single_file_size_bytes = max_single_file_size_bytes;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn compress_old_files(mut self, compress_old_files: bool) -> Self {
|
||||
self.compress_old_files = compress_old_files;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn gzip_compression_level(mut self, gzip_compression_level: u32) -> Self {
|
||||
self.gzip_compression_level = gzip_compression_level;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn compressed_file_retention_days(mut self, days: u64) -> Self {
|
||||
self.compressed_file_retention_days = days;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn exclude_patterns(mut self, patterns: Vec<String>) -> Self {
|
||||
self.exclude_patterns = patterns;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn delete_empty_files(mut self, delete_empty_files: bool) -> Self {
|
||||
self.delete_empty_files = delete_empty_files;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn min_file_age_seconds(mut self, seconds: u64) -> Self {
|
||||
self.min_file_age_seconds = seconds;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn dry_run(mut self, dry_run: bool) -> Self {
|
||||
self.dry_run = dry_run;
|
||||
self
|
||||
}
|
||||
|
||||
pub fn build(self) -> LogCleaner {
|
||||
let patterns = self
|
||||
.exclude_patterns
|
||||
.into_iter()
|
||||
.filter_map(|p| glob::Pattern::new(&p).ok())
|
||||
.collect();
|
||||
|
||||
LogCleaner {
|
||||
log_dir: self.log_dir,
|
||||
file_pattern: self.file_pattern,
|
||||
active_filename: self.active_filename,
|
||||
match_mode: self.match_mode,
|
||||
keep_files: self.keep_files,
|
||||
max_total_size_bytes: self.max_total_size_bytes,
|
||||
max_single_file_size_bytes: self.max_single_file_size_bytes,
|
||||
compress_old_files: self.compress_old_files,
|
||||
gzip_compression_level: self.gzip_compression_level.clamp(1, 9),
|
||||
compressed_file_retention_days: self.compressed_file_retention_days,
|
||||
exclude_patterns: patterns,
|
||||
delete_empty_files: self.delete_empty_files,
|
||||
min_file_age_seconds: self.min_file_age_seconds,
|
||||
dry_run: self.dry_run,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -33,21 +33,23 @@
|
||||
//! use rustfs_obs::LogCleaner;
|
||||
//! use rustfs_obs::types::FileMatchMode;
|
||||
//!
|
||||
//! let cleaner = LogCleaner::new(
|
||||
//! let cleaner = LogCleaner::builder(
|
||||
//! PathBuf::from("/var/log/rustfs"),
|
||||
//! "rustfs.log.".to_string(),
|
||||
//! FileMatchMode::Prefix,
|
||||
//! 10, // keep_files
|
||||
//! 2 * 1024 * 1024 * 1024, // max_total_size_bytes (2 GiB)
|
||||
//! 0, // max_single_file_size_bytes (unlimited)
|
||||
//! true, // compress_old_files
|
||||
//! 6, // gzip_compression_level
|
||||
//! 30, // compressed_file_retention_days
|
||||
//! vec![], // exclude_patterns
|
||||
//! true, // delete_empty_files
|
||||
//! 3600, // min_file_age_seconds (1 hour)
|
||||
//! false, // dry_run
|
||||
//! );
|
||||
//! "rustfs.log".to_string(),
|
||||
//! )
|
||||
//! .match_mode(FileMatchMode::Prefix)
|
||||
//! .keep_files(10)
|
||||
//! .max_total_size_bytes(2 * 1024 * 1024 * 1024) // 2 GiB
|
||||
//! .max_single_file_size_bytes(0) // unlimited
|
||||
//! .compress_old_files(true)
|
||||
//! .gzip_compression_level(6)
|
||||
//! .compressed_file_retention_days(30)
|
||||
//! .exclude_patterns(vec![])
|
||||
//! .delete_empty_files(true)
|
||||
//! .min_file_age_seconds(3600) // 1 hour
|
||||
//! .dry_run(false)
|
||||
//! .build();
|
||||
//!
|
||||
//! let (deleted, freed_bytes) = cleaner.cleanup().expect("cleanup failed");
|
||||
//! println!("Deleted {deleted} files, freed {freed_bytes} bytes");
|
||||
@@ -79,21 +81,13 @@ mod tests {
|
||||
|
||||
/// Build a cleaner with sensible test defaults (no compression, no age gate).
|
||||
fn make_cleaner(dir: std::path::PathBuf, keep: usize, max_bytes: u64) -> LogCleaner {
|
||||
LogCleaner::new(
|
||||
dir,
|
||||
"app.log.".to_string(),
|
||||
FileMatchMode::Prefix,
|
||||
keep,
|
||||
max_bytes,
|
||||
0, // max_single_file_size_bytes
|
||||
false, // compress_old_files
|
||||
6, // gzip_compression_level
|
||||
30, // compressed_file_retention_days
|
||||
Vec::new(), // exclude_patterns
|
||||
true, // delete_empty_files
|
||||
0, // min_file_age_seconds (0 = no age gate in tests)
|
||||
false, // dry_run
|
||||
)
|
||||
LogCleaner::builder(dir, "app.log.".to_string(), "app.log".to_string())
|
||||
.match_mode(FileMatchMode::Prefix)
|
||||
.keep_files(keep)
|
||||
.max_total_size_bytes(max_bytes)
|
||||
.min_file_age_seconds(0) // 0 = no age gate in tests
|
||||
.delete_empty_files(true)
|
||||
.build()
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -141,11 +135,15 @@ mod tests {
|
||||
create_log_file(&dir, "app.log.2024-01-02", 1024)?;
|
||||
create_log_file(&dir, "other.log", 512)?; // different prefix
|
||||
|
||||
let cleaner = make_cleaner(dir.clone(), 1, 512);
|
||||
// keep_files=1 and max_bytes=1500: deleting one managed file (1024 bytes) leaves
|
||||
// a single managed file of 1024 bytes, which satisfies both the file-count and
|
||||
// size limits. "other.log" (different prefix) must never be touched.
|
||||
let cleaner = make_cleaner(dir.clone(), 1, 1500);
|
||||
let (deleted, _) = cleaner.cleanup()?;
|
||||
|
||||
// "other.log" must not be counted or deleted.
|
||||
// "other.log" must not be counted or deleted; only 1 managed file removed.
|
||||
assert_eq!(deleted, 1, "only managed files should be deleted");
|
||||
assert!(dir.join("other.log").exists(), "unrelated file must not be deleted");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -158,8 +156,8 @@ mod tests {
|
||||
create_log_file(&dir, "app.log.2024-01-02", 2048)?;
|
||||
create_log_file(&dir, "other.log", 512)?;
|
||||
|
||||
let files = scanner::collect_log_files(&dir, "app.log.", FileMatchMode::Prefix, &[], 0, true, false)?;
|
||||
assert_eq!(files.len(), 2, "scanner should find exactly 2 managed files");
|
||||
let result = scanner::scan_log_directory(&dir, "app.log.", Some("app.log"), FileMatchMode::Prefix, &[], 0, true, false)?;
|
||||
assert_eq!(result.logs.len(), 2, "scanner should find exactly 2 managed files");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -172,21 +170,12 @@ mod tests {
|
||||
create_log_file(&dir, "app.log.2024-01-02", 1024)?;
|
||||
create_log_file(&dir, "app.log.2024-01-03", 1024)?;
|
||||
|
||||
let cleaner = LogCleaner::new(
|
||||
dir.clone(),
|
||||
"app.log.".to_string(),
|
||||
FileMatchMode::Prefix,
|
||||
1,
|
||||
1024,
|
||||
0,
|
||||
false,
|
||||
6,
|
||||
30,
|
||||
vec![],
|
||||
true,
|
||||
0,
|
||||
true,
|
||||
);
|
||||
let cleaner = LogCleaner::builder(dir.clone(), "app.log.".to_string(), "app.log".to_string())
|
||||
.match_mode(FileMatchMode::Prefix)
|
||||
.keep_files(1)
|
||||
.max_total_size_bytes(1024)
|
||||
.dry_run(true)
|
||||
.build();
|
||||
let (deleted, _freed) = cleaner.cleanup()?;
|
||||
|
||||
// dry_run=true reports deletions but doesn't actually remove files.
|
||||
@@ -204,21 +193,11 @@ mod tests {
|
||||
create_log_file(&dir, "2026-03-01-06-22.rustfs.log", 1024)?;
|
||||
create_log_file(&dir, "other.log", 1024)?; // not managed
|
||||
|
||||
let cleaner = LogCleaner::new(
|
||||
dir.clone(),
|
||||
"rustfs.log".to_string(),
|
||||
FileMatchMode::Suffix,
|
||||
1,
|
||||
1024,
|
||||
0,
|
||||
false,
|
||||
6,
|
||||
30,
|
||||
vec![],
|
||||
true,
|
||||
0,
|
||||
false,
|
||||
);
|
||||
let cleaner = LogCleaner::builder(dir.clone(), ".rustfs.log".to_string(), "current.log".to_string())
|
||||
.match_mode(FileMatchMode::Suffix)
|
||||
.keep_files(1)
|
||||
.max_total_size_bytes(1024)
|
||||
.build();
|
||||
let (deleted, freed) = cleaner.cleanup()?;
|
||||
|
||||
assert_eq!(deleted, 1, "should delete exactly one file");
|
||||
|
||||
@@ -14,55 +14,91 @@
|
||||
|
||||
//! Filesystem scanner for discovering log files eligible for cleanup.
|
||||
//!
|
||||
//! This module is intentionally kept read-only: it does **not** delete or
|
||||
//! compress any files — it only reports what it found.
|
||||
//! This module is primarily read-only: it reports what files it found.
|
||||
//! The one exception is zero-byte file removal — when `delete_empty_files`
|
||||
//! is enabled, `scan_log_directory` removes empty regular files as part of
|
||||
//! the scan so that they are not counted in retention calculations.
|
||||
|
||||
use super::types::{FileInfo, FileMatchMode};
|
||||
use rustfs_config::observability::DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION;
|
||||
use std::fs;
|
||||
use std::path::Path;
|
||||
use std::time::{Duration, SystemTime};
|
||||
use std::time::SystemTime;
|
||||
use tracing::debug;
|
||||
use walkdir::WalkDir;
|
||||
|
||||
/// Collect all log files in `log_dir` whose name matches `file_pattern` based on `match_mode`.
|
||||
/// Result of a single pass directory scan.
|
||||
pub(super) struct LogScanResult {
|
||||
/// Regular log files eligible for deletion/compression.
|
||||
pub logs: Vec<FileInfo>,
|
||||
/// Already compressed files eligible for expiry deletion.
|
||||
pub compressed_archives: Vec<FileInfo>,
|
||||
}
|
||||
|
||||
/// Perform a single-pass scan of the log directory.
|
||||
///
|
||||
/// Files that:
|
||||
/// - are already compressed (`.gz` extension),
|
||||
/// - are zero-byte and `delete_empty_files` is `true` (these are handled
|
||||
/// immediately by the caller), or
|
||||
/// - match one of the `exclude_patterns`,
|
||||
/// - were modified more recently than `min_file_age_seconds` seconds ago,
|
||||
///
|
||||
/// are skipped and not returned in the result list.
|
||||
/// This function iterates over the directory entries once and categorizes them
|
||||
/// into regular logs or compressed archives based on extensions and patterns.
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `log_dir` - Root directory to scan (depth 1 only, no recursion).
|
||||
/// * `file_pattern` - Pattern string to match filenames.
|
||||
/// * `active_filename` - The name of the currently active log file (to be excluded).
|
||||
/// * `match_mode` - Whether to match by prefix or suffix.
|
||||
/// * `exclude_patterns` - Compiled glob patterns; matching files are skipped.
|
||||
/// * `min_file_age_seconds` - Files younger than this threshold are skipped.
|
||||
/// * `delete_empty_files` - When `true`, zero-byte files trigger an immediate
|
||||
/// delete by the caller before the rest of cleanup runs.
|
||||
pub(super) fn collect_log_files(
|
||||
/// * `min_file_age_seconds` - Files younger than this threshold are skipped (for regular logs).
|
||||
/// * `delete_empty_files` - When `true`, zero-byte regular files that match
|
||||
/// the pattern are deleted immediately inside this function and excluded
|
||||
/// from the returned [`LogScanResult`].
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub(super) fn scan_log_directory(
|
||||
log_dir: &Path,
|
||||
file_pattern: &str,
|
||||
active_filename: Option<&str>,
|
||||
match_mode: FileMatchMode,
|
||||
exclude_patterns: &[glob::Pattern],
|
||||
min_file_age_seconds: u64,
|
||||
delete_empty_files: bool,
|
||||
dry_run: bool,
|
||||
) -> Result<Vec<FileInfo>, std::io::Error> {
|
||||
let mut files = Vec::new();
|
||||
) -> Result<LogScanResult, std::io::Error> {
|
||||
let mut logs = Vec::new();
|
||||
let mut compressed_archives = Vec::new();
|
||||
let now = SystemTime::now();
|
||||
|
||||
for entry in WalkDir::new(log_dir)
|
||||
.max_depth(1)
|
||||
.follow_links(false)
|
||||
.into_iter()
|
||||
.filter_map(|e| e.ok())
|
||||
{
|
||||
// Use read_dir for a lightweight, non-recursive scan.
|
||||
let entries = match fs::read_dir(log_dir) {
|
||||
Ok(entries) => entries,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
|
||||
// If the log directory does not exist (or was removed), treat this
|
||||
// as "no files found" instead of failing the whole cleanup pass.
|
||||
return Ok(LogScanResult {
|
||||
logs,
|
||||
compressed_archives,
|
||||
});
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
};
|
||||
|
||||
for entry in entries {
|
||||
let entry = match entry {
|
||||
Ok(e) => e,
|
||||
Err(_) => continue, // Skip unreadable entries
|
||||
};
|
||||
|
||||
let path = entry.path();
|
||||
if !path.is_file() {
|
||||
|
||||
// We only care about regular files inside the log directory.
|
||||
// Use `fs::symlink_metadata` (which does *not* follow symlinks) for
|
||||
// both the file-type check *and* size/mtime collection below. Using
|
||||
// `entry.metadata()` or `Path::is_file()` (both of which follow
|
||||
// symlinks) would allow a symlink placed in the log directory to reach
|
||||
// files outside the tree, and would introduce a TOCTOU window between
|
||||
// the type-check and the metadata read.
|
||||
let metadata = match fs::symlink_metadata(&path) {
|
||||
Ok(md) => md,
|
||||
Err(_) => continue,
|
||||
};
|
||||
let file_type = metadata.file_type();
|
||||
if !file_type.is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
@@ -71,42 +107,53 @@ pub(super) fn collect_log_files(
|
||||
None => continue,
|
||||
};
|
||||
|
||||
// Match filename based on mode
|
||||
// 1. Explicitly skip the active log file (if known).
|
||||
if let Some(active) = active_filename
|
||||
&& filename == active
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// 2. Check exclusion patterns early.
|
||||
if is_excluded(filename, exclude_patterns) {
|
||||
debug!("Excluding file from cleanup: {:?}", filename);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 3. Classify file type and check pattern match.
|
||||
let is_compressed = filename.ends_with(DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION);
|
||||
|
||||
// For matching, we need the "base" name.
|
||||
// If compressed: "foo.log.gz" -> check "foo.log"
|
||||
// If regular: "foo.log" -> check "foo.log"
|
||||
let name_to_match = if is_compressed {
|
||||
&filename[..filename.len() - DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION.len()]
|
||||
} else {
|
||||
filename
|
||||
};
|
||||
|
||||
let matches = match match_mode {
|
||||
FileMatchMode::Prefix => filename.starts_with(file_pattern),
|
||||
FileMatchMode::Suffix => filename.ends_with(file_pattern),
|
||||
FileMatchMode::Prefix => name_to_match.starts_with(file_pattern),
|
||||
FileMatchMode::Suffix => name_to_match.ends_with(file_pattern),
|
||||
};
|
||||
|
||||
if !matches {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Compressed files are handled by collect_compressed_files.
|
||||
if filename.ends_with(DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Honour exclusion patterns.
|
||||
if is_excluded(filename, exclude_patterns) {
|
||||
debug!("Excluding file from cleanup: {:?}", filename);
|
||||
continue;
|
||||
}
|
||||
|
||||
let metadata = match entry.metadata() {
|
||||
Ok(m) => m,
|
||||
Err(_) => continue,
|
||||
};
|
||||
// 4. Gather size and mtime from the already-fetched symlink_metadata
|
||||
// (reuse; no second syscall, no symlink following).
|
||||
let file_size = metadata.len();
|
||||
let modified = match metadata.modified() {
|
||||
Ok(t) => t,
|
||||
Err(_) => continue,
|
||||
Err(_) => continue, // Skip files where we can't read modification time
|
||||
};
|
||||
let file_size = metadata.len();
|
||||
|
||||
// Delete zero-byte files immediately (outside the normal selection
|
||||
// logic) when the feature is enabled.
|
||||
if file_size == 0 && delete_empty_files {
|
||||
// 5. Handle zero-byte files (Regular logs only).
|
||||
// We generally don't delete empty compressed files implicitly, but let's stick to regular files logic.
|
||||
if !is_compressed && file_size == 0 && delete_empty_files {
|
||||
if !dry_run {
|
||||
if let Err(e) = std::fs::remove_file(path) {
|
||||
if let Err(e) = std::fs::remove_file(&path) {
|
||||
tracing::warn!("Failed to delete empty file {:?}: {}", path, e);
|
||||
} else {
|
||||
debug!("Deleted empty file: {:?}", path);
|
||||
@@ -117,99 +164,33 @@ pub(super) fn collect_log_files(
|
||||
continue;
|
||||
}
|
||||
|
||||
// Skip files that are too young.
|
||||
if let Ok(age) = now.duration_since(modified)
|
||||
// 6. Age Check (Regular logs only).
|
||||
// Compressed files have their own retention check in the caller.
|
||||
if !is_compressed
|
||||
&& let Ok(age) = now.duration_since(modified)
|
||||
&& age.as_secs() < min_file_age_seconds
|
||||
{
|
||||
debug!(
|
||||
"Skipping file (too new): {:?}, age: {}s, min_age: {}s",
|
||||
filename,
|
||||
age.as_secs(),
|
||||
min_file_age_seconds
|
||||
);
|
||||
// Too young to be touched.
|
||||
continue;
|
||||
}
|
||||
|
||||
files.push(FileInfo {
|
||||
path: path.to_path_buf(),
|
||||
let info = FileInfo {
|
||||
path,
|
||||
size: file_size,
|
||||
modified,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
}
|
||||
|
||||
/// Collect compressed `.gz` log files whose age exceeds the retention period.
|
||||
///
|
||||
/// When `compressed_file_retention_days` is `0` the function returns immediately
|
||||
/// without collecting anything (files are kept indefinitely).
|
||||
///
|
||||
/// # Arguments
|
||||
/// * `log_dir` - Root directory to scan.
|
||||
/// * `file_pattern` - Pattern string to match filenames.
|
||||
/// * `match_mode` - Whether to match by prefix or suffix.
|
||||
/// * `compressed_file_retention_days` - Files older than this are eligible for
|
||||
/// deletion; `0` means never delete compressed files.
|
||||
pub(super) fn collect_expired_compressed_files(
|
||||
log_dir: &Path,
|
||||
file_pattern: &str,
|
||||
match_mode: FileMatchMode,
|
||||
compressed_file_retention_days: u64,
|
||||
) -> Result<Vec<FileInfo>, std::io::Error> {
|
||||
if compressed_file_retention_days == 0 {
|
||||
return Ok(Vec::new());
|
||||
}
|
||||
|
||||
let retention = Duration::from_secs(compressed_file_retention_days * 24 * 3600);
|
||||
let now = SystemTime::now();
|
||||
let mut files = Vec::new();
|
||||
|
||||
for entry in WalkDir::new(log_dir)
|
||||
.max_depth(1)
|
||||
.follow_links(false)
|
||||
.into_iter()
|
||||
.filter_map(|e| e.ok())
|
||||
{
|
||||
let path = entry.path();
|
||||
if !path.is_file() {
|
||||
continue;
|
||||
}
|
||||
|
||||
let filename = match path.file_name().and_then(|n| n.to_str()) {
|
||||
Some(f) => f,
|
||||
None => continue,
|
||||
};
|
||||
|
||||
if !filename.ends_with(DEFAULT_OBS_LOG_GZIP_COMPRESSION_ALL_EXTENSION) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if the base filename (without .gz) matches the pattern
|
||||
let base_filename = &filename[..filename.len() - 3];
|
||||
let matches = match match_mode {
|
||||
FileMatchMode::Prefix => base_filename.starts_with(file_pattern),
|
||||
FileMatchMode::Suffix => base_filename.ends_with(file_pattern),
|
||||
};
|
||||
|
||||
if !matches {
|
||||
continue;
|
||||
}
|
||||
|
||||
let Ok(metadata) = entry.metadata() else { continue };
|
||||
let Ok(modified) = metadata.modified() else { continue };
|
||||
let Ok(age) = now.duration_since(modified) else { continue };
|
||||
|
||||
if age > retention {
|
||||
files.push(FileInfo {
|
||||
path: path.to_path_buf(),
|
||||
size: metadata.len(),
|
||||
modified,
|
||||
});
|
||||
if is_compressed {
|
||||
compressed_archives.push(info);
|
||||
} else {
|
||||
logs.push(info);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(files)
|
||||
Ok(LogScanResult {
|
||||
logs,
|
||||
compressed_archives,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns `true` if `filename` matches any of the compiled exclusion patterns.
|
||||
|
||||
@@ -210,7 +210,11 @@ impl OtelConfig {
|
||||
|
||||
// `log_keep_files` is the single source of truth for file retention count.
|
||||
// It defaults to `DEFAULT_LOG_KEEP_FILES` (30).
|
||||
let log_keep_files = Some(get_env_usize(ENV_OBS_LOG_KEEP_FILES, DEFAULT_LOG_KEEP_FILES));
|
||||
let mut log_keep_files = get_env_usize(ENV_OBS_LOG_KEEP_FILES, DEFAULT_LOG_KEEP_FILES);
|
||||
if log_keep_files == 0 {
|
||||
log_keep_files = DEFAULT_LOG_KEEP_FILES;
|
||||
}
|
||||
let log_keep_files = Some(log_keep_files);
|
||||
|
||||
// `log_rotation_time` drives the rolling-appender rotation period.
|
||||
let log_rotation_time = Some(get_env_str(ENV_OBS_LOG_ROTATION_TIME, DEFAULT_LOG_ROTATION_TIME));
|
||||
|
||||
@@ -75,12 +75,25 @@ pub(super) fn build_env_filter(logger_level: &str, default_level: Option<&str>)
|
||||
.map(EnvFilter::new)
|
||||
.unwrap_or_else(|| EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new(level)));
|
||||
|
||||
// Suppress chatty infrastructure crates unless the operator explicitly
|
||||
// requests trace/debug output.
|
||||
if should_suppress_noisy_crates(logger_level, default_level, rust_log.as_deref()) {
|
||||
let directives: SmallVec<[&str; 5]> = smallvec::smallvec!["hyper", "tonic", "h2", "reqwest", "tower"];
|
||||
for directive in directives {
|
||||
filter = filter.add_directive(format!("{directive}=off").parse().unwrap());
|
||||
let directives: SmallVec<[(&str, &str); 6]> = smallvec::smallvec![
|
||||
("hyper", "off"),
|
||||
("tonic", "off"),
|
||||
("h2", "off"),
|
||||
("reqwest", "off"),
|
||||
("tower", "off"),
|
||||
// HTTP request logs are demoted to WARN to reduce volume in production.
|
||||
("rustfs::server::http", "warn"),
|
||||
];
|
||||
for (crate_name, level) in directives {
|
||||
match format!("{crate_name}={level}").parse() {
|
||||
Ok(directive) => filter = filter.add_directive(directive),
|
||||
Err(e) => {
|
||||
// The directive strings are compile-time constants, so this
|
||||
// branch should never be reached; emit a diagnostic just in case.
|
||||
eprintln!("obs: invalid log filter directive '{crate_name}={level}': {e}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -32,6 +32,7 @@ use crate::cleaner::types::FileMatchMode;
|
||||
use crate::config::OtelConfig;
|
||||
use crate::global::OBSERVABILITY_METRIC_ENABLED;
|
||||
use crate::telemetry::filter::build_env_filter;
|
||||
use crate::telemetry::rolling::{RollingAppender, Rotation};
|
||||
use metrics::counter;
|
||||
use rustfs_config::observability::{
|
||||
DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS, DEFAULT_OBS_LOG_COMPRESS_OLD_FILES, DEFAULT_OBS_LOG_COMPRESSED_FILE_RETENTION_DAYS,
|
||||
@@ -175,7 +176,7 @@ fn init_file_logging_internal(
|
||||
|
||||
// ── 3. Choose rotation strategy ──────────────────────────────────────────
|
||||
// `log_rotation_time` drives the rolling-appender rotation period.
|
||||
let rotation = config
|
||||
let rotation_str = config
|
||||
.log_rotation_time
|
||||
.as_deref()
|
||||
.unwrap_or(DEFAULT_LOG_ROTATION_TIME)
|
||||
@@ -187,28 +188,20 @@ fn init_file_logging_internal(
|
||||
_ => FileMatchMode::Suffix,
|
||||
};
|
||||
|
||||
use tracing_appender::rolling::{RollingFileAppender, Rotation};
|
||||
let file_appender = {
|
||||
let rotation = match rotation.as_str() {
|
||||
"minutely" => Rotation::MINUTELY,
|
||||
"hourly" => Rotation::HOURLY,
|
||||
_ => Rotation::DAILY,
|
||||
};
|
||||
|
||||
let mut builder = RollingFileAppender::builder()
|
||||
.rotation(rotation)
|
||||
.max_log_files(keep_files * 3); // Make sure there are some data files to archive to avoid premature deletion
|
||||
|
||||
match match_mode {
|
||||
FileMatchMode::Prefix => builder = builder.filename_prefix(log_filename),
|
||||
FileMatchMode::Suffix => builder = builder.filename_suffix(log_filename),
|
||||
}
|
||||
|
||||
builder
|
||||
.build(log_directory)
|
||||
.map_err(|e| TelemetryError::Io(format!("failed to initialize rolling file appender: {e}")))?
|
||||
let rotation = match rotation_str.as_str() {
|
||||
"minutely" => Rotation::Minutely,
|
||||
"hourly" => Rotation::Hourly,
|
||||
"daily" => Rotation::Daily,
|
||||
_ => Rotation::Daily,
|
||||
};
|
||||
|
||||
let max_single_file_size = config
|
||||
.log_max_single_file_size_bytes
|
||||
.unwrap_or(DEFAULT_OBS_LOG_MAX_SINGLE_FILE_SIZE_BYTES);
|
||||
|
||||
let file_appender =
|
||||
RollingAppender::new(log_directory, log_filename.to_string(), rotation, max_single_file_size, match_mode)?;
|
||||
|
||||
let (non_blocking, guard) = tracing_appender::non_blocking(file_appender);
|
||||
|
||||
// ── 4. Build subscriber layers ────────────────────────────────────────────
|
||||
@@ -270,7 +263,7 @@ fn init_file_logging_internal(
|
||||
|
||||
info!(
|
||||
"Init file logging at '{}', rotation: {}, keep {} files",
|
||||
log_directory, rotation, keep_files
|
||||
log_directory, rotation_str, keep_files
|
||||
);
|
||||
|
||||
Ok(OtelGuard {
|
||||
@@ -352,6 +345,7 @@ fn spawn_cleanup_task(
|
||||
// Use suffix matching for log files like "2026-03-01-06-21.rustfs.log"
|
||||
// where "rustfs.log" is the suffix.
|
||||
let file_pattern = config.log_filename.as_deref().unwrap_or(log_filename).to_string();
|
||||
let active_filename = file_pattern.clone();
|
||||
|
||||
// Determine match mode from config, defaulting to Suffix
|
||||
let match_mode = match config.log_match_mode.as_deref().map(|s| s.to_lowercase()).as_deref() {
|
||||
@@ -386,21 +380,21 @@ fn spawn_cleanup_task(
|
||||
.log_cleanup_interval_seconds
|
||||
.unwrap_or(DEFAULT_OBS_LOG_CLEANUP_INTERVAL_SECONDS);
|
||||
|
||||
let cleaner = Arc::new(LogCleaner::new(
|
||||
log_dir,
|
||||
file_pattern,
|
||||
match_mode,
|
||||
keep_files,
|
||||
max_total_size,
|
||||
max_single_file_size,
|
||||
compress,
|
||||
gzip_level,
|
||||
retention_days,
|
||||
exclude_patterns,
|
||||
delete_empty,
|
||||
min_age,
|
||||
dry_run,
|
||||
));
|
||||
let cleaner = Arc::new(
|
||||
LogCleaner::builder(log_dir, file_pattern, active_filename)
|
||||
.match_mode(match_mode)
|
||||
.keep_files(keep_files)
|
||||
.max_total_size_bytes(max_total_size)
|
||||
.max_single_file_size_bytes(max_single_file_size)
|
||||
.compress_old_files(compress)
|
||||
.gzip_compression_level(gzip_level)
|
||||
.compressed_file_retention_days(retention_days)
|
||||
.exclude_patterns(exclude_patterns)
|
||||
.delete_empty_files(delete_empty)
|
||||
.min_file_age_seconds(min_age)
|
||||
.dry_run(dry_run)
|
||||
.build(),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(cleanup_interval));
|
||||
@@ -433,8 +427,13 @@ mod tests {
|
||||
..OtelConfig::default()
|
||||
};
|
||||
|
||||
let result = init_file_logging_internal(&config, temp_path, "info", true);
|
||||
|
||||
assert!(result.is_err());
|
||||
// We must run within a Tokio runtime because init_file_logging_internal spawns a background task.
|
||||
let rt = tokio::runtime::Runtime::new().unwrap();
|
||||
rt.block_on(async {
|
||||
let result = init_file_logging_internal(&config, temp_path, "info", true);
|
||||
// With eager file opening, an invalid filename (null byte) causes the OS to reject
|
||||
// the open() call, so the function returns Err instead of panicking.
|
||||
assert!(result.is_err(), "invalid filename must return Err, not panic");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -45,6 +45,7 @@ mod local;
|
||||
mod otel;
|
||||
mod recorder;
|
||||
mod resource;
|
||||
mod rolling;
|
||||
|
||||
use crate::TelemetryError;
|
||||
use crate::config::OtelConfig;
|
||||
@@ -117,8 +118,9 @@ pub(crate) fn init_telemetry(config: &OtelConfig) -> Result<OtelGuard, Telemetry
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use rustfs_config::observability::DEFAULT_OBS_ENVIRONMENT_PRODUCTION;
|
||||
use rustfs_config::{ENVIRONMENT, USE_STDOUT};
|
||||
use rustfs_config::{DEFAULT_OBS_LOG_STDOUT_ENABLED, ENVIRONMENT};
|
||||
|
||||
#[test]
|
||||
fn test_production_environment_detection() {
|
||||
@@ -160,7 +162,7 @@ mod tests {
|
||||
TestCase {
|
||||
is_production: false,
|
||||
config_use_stdout: None,
|
||||
expected_use_stdout: USE_STDOUT,
|
||||
expected_use_stdout: DEFAULT_OBS_LOG_STDOUT_ENABLED,
|
||||
description: "Non-production with no config should use default",
|
||||
},
|
||||
TestCase {
|
||||
@@ -184,7 +186,11 @@ mod tests {
|
||||
];
|
||||
|
||||
for case in &test_cases {
|
||||
let default_use_stdout = if case.is_production { false } else { USE_STDOUT };
|
||||
let default_use_stdout = if case.is_production {
|
||||
false
|
||||
} else {
|
||||
DEFAULT_OBS_LOG_STDOUT_ENABLED
|
||||
};
|
||||
let actual = case.config_use_stdout.unwrap_or(default_use_stdout);
|
||||
assert_eq!(actual, case.expected_use_stdout, "Test case failed: {}", case.description);
|
||||
}
|
||||
@@ -221,7 +227,6 @@ mod tests {
|
||||
#[test]
|
||||
fn test_otel_config_environment_defaults() {
|
||||
// Verify that environment field defaults behave correctly.
|
||||
use crate::config::OtelConfig;
|
||||
let config = OtelConfig {
|
||||
endpoint: "".to_string(),
|
||||
use_stdout: None,
|
||||
|
||||
508
crates/obs/src/telemetry/rolling.rs
Normal file
508
crates/obs/src/telemetry/rolling.rs
Normal file
@@ -0,0 +1,508 @@
|
||||
// Copyright 2024 RustFS Team
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// http://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
|
||||
//! A custom rolling file appender that supports both time-based and size-based rotation.
|
||||
//!
|
||||
//! This is a lightweight replacement for `tracing_appender::rolling::RollingFileAppender`
|
||||
//! which only supports time-based rotation. This implementation ensures that active
|
||||
//! log files do not grow indefinitely by rotating them when they exceed a configured size.
|
||||
|
||||
use crate::cleaner::types::FileMatchMode;
|
||||
use jiff::Zoned;
|
||||
use std::fs::{self, File};
|
||||
use std::io::{self, Write};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU64, Ordering};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub enum Rotation {
|
||||
Minutely,
|
||||
Hourly,
|
||||
Daily,
|
||||
#[allow(dead_code)]
|
||||
Never,
|
||||
}
|
||||
|
||||
/// Global per-process counter used to disambiguate archive filenames that
|
||||
/// may otherwise collide when multiple rotations occur within the same
|
||||
/// timestamp tick.
|
||||
static ROLL_UNIQUIFIER: AtomicU64 = AtomicU64::new(0);
|
||||
|
||||
impl Rotation {
|
||||
fn check_should_roll(&self, last: i64, now: i64) -> bool {
|
||||
match self {
|
||||
Rotation::Minutely => now / 60 != last / 60,
|
||||
Rotation::Hourly => now / 3600 != last / 3600,
|
||||
Rotation::Daily => {
|
||||
// Align daily rotation with the local day boundary rather than UTC midnight.
|
||||
// We shift both timestamps by the current local offset before bucketing into days.
|
||||
let offset_secs = Zoned::now().offset().seconds() as i64;
|
||||
(now + offset_secs) / 86400 != (last + offset_secs) / 86400
|
||||
}
|
||||
Rotation::Never => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub struct RollingAppender {
|
||||
dir: PathBuf,
|
||||
filename: String,
|
||||
rotation: Rotation,
|
||||
max_size_bytes: u64,
|
||||
match_mode: FileMatchMode,
|
||||
|
||||
file: Option<File>,
|
||||
size: u64,
|
||||
// Store as seconds since Unix epoch
|
||||
last_roll_ts: i64,
|
||||
}
|
||||
|
||||
impl RollingAppender {
|
||||
/// Create and immediately validate a new `RollingAppender`.
|
||||
///
|
||||
/// The log directory is created if it does not already exist, and the
|
||||
/// active log file is opened (or created) eagerly so that configuration
|
||||
/// errors — e.g. an invalid filename — surface at initialisation time
|
||||
/// rather than on the first write.
|
||||
///
|
||||
/// # Errors
|
||||
/// Returns an [`io::Error`] if:
|
||||
/// - `filename` is not a plain file name (absolute path, path separators,
|
||||
/// or `..` components are rejected to prevent path traversal).
|
||||
/// - The directory cannot be created.
|
||||
/// - The active log file cannot be opened/created.
|
||||
pub fn new(
|
||||
dir: impl AsRef<Path>,
|
||||
filename: String,
|
||||
rotation: Rotation,
|
||||
max_size_bytes: u64,
|
||||
match_mode: FileMatchMode,
|
||||
) -> io::Result<Self> {
|
||||
// Validate that `filename` is a plain file name: not absolute and no
|
||||
// directory components (separators or `..`). If `file_name()` equals
|
||||
// the entire path, there can be no parent-directory traversal.
|
||||
{
|
||||
let p = Path::new(&filename);
|
||||
let is_plain_name = !p.is_absolute() && p.file_name().map(|n| n == p.as_os_str()).unwrap_or(false);
|
||||
if !is_plain_name {
|
||||
return Err(io::Error::new(
|
||||
io::ErrorKind::InvalidInput,
|
||||
format!("log filename must be a plain file name with no path components, got: {filename:?}"),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let mut appender = Self {
|
||||
dir: dir.as_ref().to_path_buf(),
|
||||
filename,
|
||||
rotation,
|
||||
max_size_bytes,
|
||||
match_mode,
|
||||
file: None,
|
||||
size: 0,
|
||||
last_roll_ts: Zoned::now().timestamp().as_second(),
|
||||
};
|
||||
// Eagerly open the file to validate the path and capture accurate
|
||||
// initial size / last-roll timestamp.
|
||||
appender.open_file()?;
|
||||
Ok(appender)
|
||||
}
|
||||
|
||||
fn active_file_path(&self) -> PathBuf {
|
||||
self.dir.join(&self.filename)
|
||||
}
|
||||
|
||||
fn open_file(&mut self) -> io::Result<()> {
|
||||
if self.file.is_some() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let path = self.active_file_path();
|
||||
// Ensure directory exists
|
||||
if let Some(parent) = path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
// Open in append mode
|
||||
let file = fs::OpenOptions::new().create(true).append(true).open(&path)?;
|
||||
|
||||
let meta = file.metadata()?;
|
||||
self.size = meta.len();
|
||||
|
||||
// Seed `last_roll_ts` from the file's modification time so that a
|
||||
// process restart correctly triggers time-based rotation if the active
|
||||
// file belongs to a previous period.
|
||||
if let Ok(modified) = meta.modified() {
|
||||
// Convert SystemTime to jiff::Timestamp
|
||||
if let Ok(ts) = jiff::Timestamp::try_from(modified) {
|
||||
self.last_roll_ts = ts.as_second();
|
||||
}
|
||||
}
|
||||
|
||||
self.file = Some(file);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn should_roll(&self, write_len: u64) -> bool {
|
||||
// 1. Size-based check (Cheap, check first)
|
||||
// If max_size is set (non-zero) and writing would exceed it, roll immediately.
|
||||
if self.max_size_bytes > 0 && (self.size + write_len) > self.max_size_bytes {
|
||||
return true;
|
||||
}
|
||||
|
||||
// 2. Time-based check
|
||||
// We check this after size check to avoid unnecessary time calls if size forces a roll.
|
||||
let now = Zoned::now().timestamp().as_second();
|
||||
|
||||
self.rotation.check_should_roll(self.last_roll_ts, now)
|
||||
}
|
||||
|
||||
fn roll(&mut self) -> io::Result<()> {
|
||||
// 1. Close current file first to ensure all buffers are flushed to OS (if any)
|
||||
// and handle released.
|
||||
self.file = None;
|
||||
|
||||
let active_path = self.active_file_path();
|
||||
if !active_path.exists() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// 2. Generate archive name.
|
||||
// Format: YYYYMMDDHHMMSS.uuuuuu (Microsecond/Nanosecond precision)
|
||||
// We use jiff's strftime. "%Y%m%d%H%M%S%.6f" gives microsecond precision.
|
||||
let now = Zoned::now();
|
||||
let timestamp_str = now.strftime("%Y%m%d%H%M%S%.6f").to_string();
|
||||
|
||||
// Add a unique counter to prevent collisions in high-concurrency/fast-rotation scenarios.
|
||||
let counter = ROLL_UNIQUIFIER.fetch_add(1, Ordering::Relaxed);
|
||||
|
||||
// Final suffix/prefix part: timestamp + counter
|
||||
// Example: 20231027103001.123456-0
|
||||
let unique_part = format!("{}-{}", timestamp_str, counter);
|
||||
|
||||
// Match naming strategy with LogCleaner expectations.
|
||||
let archive_name = match self.match_mode {
|
||||
FileMatchMode::Suffix => {
|
||||
// Suffix mode: timestamp BEFORE filename.
|
||||
// e.g. rustfs.log -> 20231027103001.123456-0.rustfs.log
|
||||
format!("{}.{}", unique_part, self.filename)
|
||||
}
|
||||
FileMatchMode::Prefix => {
|
||||
// Prefix mode: timestamp AFTER filename.
|
||||
// e.g. rustfs -> rustfs.20231027103001.123456-0
|
||||
format!("{}.{}", self.filename, unique_part)
|
||||
}
|
||||
};
|
||||
|
||||
// 3. Rename the active file to the archive path.
|
||||
let archive_path = self.dir.join(&archive_name);
|
||||
|
||||
// Robust Rename Strategy:
|
||||
// On Windows, file locking (e.g. by AV software or indexers) can cause `rename` to fail
|
||||
// spuriously with PermissionDenied. We implement a short retry loop with backoff.
|
||||
const MAX_RETRIES: u32 = 3;
|
||||
let mut last_error = None;
|
||||
|
||||
for i in 0..MAX_RETRIES {
|
||||
match fs::rename(&active_path, &archive_path) {
|
||||
Ok(_) => {
|
||||
// Success!
|
||||
// 4. Reset state
|
||||
self.size = 0;
|
||||
self.last_roll_ts = now.timestamp().as_second();
|
||||
|
||||
// 5. Re-open (creates new active file)
|
||||
self.open_file()?;
|
||||
return Ok(());
|
||||
}
|
||||
Err(e) => {
|
||||
// Decide if we should retry based on error kind
|
||||
let should_retry = match e.kind() {
|
||||
// Windows often returns PermissionDenied for locked files
|
||||
io::ErrorKind::PermissionDenied => true,
|
||||
io::ErrorKind::Interrupted => true,
|
||||
_ => false,
|
||||
};
|
||||
|
||||
last_error = Some(e);
|
||||
|
||||
if !should_retry {
|
||||
break;
|
||||
}
|
||||
|
||||
// Exponential backoff: 10ms, 20ms, 40ms...
|
||||
thread::sleep(Duration::from_millis(10 * (1 << i)));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 6. Recovery Failure
|
||||
// If we exhausted retries, we MUST NOT lose log data.
|
||||
// We re-open the ACTIVE file (which is still there because rename failed).
|
||||
// The file will grow beyond max_size, but availability > strict sizing.
|
||||
eprintln!(
|
||||
"RollingAppender: Failed to rotate log file after {} retries. Error: {:?}",
|
||||
MAX_RETRIES, last_error
|
||||
);
|
||||
|
||||
// Attempt to re-open existing active file to allow continued writing
|
||||
self.open_file()?;
|
||||
|
||||
// Return the error so it can be logged/handled, even though we recovered the handle.
|
||||
Err(last_error.unwrap_or_else(|| io::Error::other("Unknown rename error")))
|
||||
}
|
||||
}
|
||||
|
||||
impl Write for RollingAppender {
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
// Ensure file is open
|
||||
if self.file.is_none() {
|
||||
self.open_file()?;
|
||||
}
|
||||
|
||||
// Check rotation
|
||||
if self.should_roll(buf.len() as u64)
|
||||
&& let Err(e) = self.roll()
|
||||
{
|
||||
// If rotation fails, we log to stderr and try to continue writing to the active file
|
||||
// to avoid losing logs if possible.
|
||||
eprintln!("RollingAppender: failed to rotate log file: {}", e);
|
||||
}
|
||||
|
||||
// Ensure file is open (in case roll closed it and failed to open new one, or open_file failed above)
|
||||
if self.file.is_none() {
|
||||
self.open_file()?;
|
||||
}
|
||||
|
||||
if let Some(file) = &mut self.file {
|
||||
let n = file.write(buf)?;
|
||||
self.size += n as u64;
|
||||
Ok(n)
|
||||
} else {
|
||||
Err(io::Error::other("Failed to open log file"))
|
||||
}
|
||||
}
|
||||
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
if let Some(file) = &mut self.file {
|
||||
file.flush()
|
||||
} else {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::fs;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn count_files(dir: &Path) -> usize {
|
||||
fs::read_dir(dir)
|
||||
.unwrap()
|
||||
.filter(|e| e.as_ref().unwrap().path().is_file())
|
||||
.count()
|
||||
}
|
||||
|
||||
// ── Construction ──────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_new_creates_file_eagerly() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let _appender = RollingAppender::new(tmp.path(), "test.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix)
|
||||
.expect("should create appender without error");
|
||||
assert!(tmp.path().join("test.log").exists(), "active log file should be created on new()");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_new_invalid_filename_returns_error() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// Null byte is invalid on both Unix and Windows.
|
||||
let result = RollingAppender::new(tmp.path(), "invalid\0name.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix);
|
||||
assert!(result.is_err(), "null byte in filename must produce an error");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_new_rejects_path_with_separators() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// A filename containing path separators could escape the log directory.
|
||||
let result = RollingAppender::new(tmp.path(), "subdir/app.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix);
|
||||
assert!(result.is_err(), "filename with path separator must be rejected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_new_rejects_parent_directory_traversal() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// "../secret.log" would write outside the log directory.
|
||||
let result = RollingAppender::new(tmp.path(), "../secret.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix);
|
||||
assert!(result.is_err(), "parent-directory traversal in filename must be rejected");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_new_rejects_absolute_path_as_filename() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let result = RollingAppender::new(tmp.path(), "/etc/app.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix);
|
||||
assert!(result.is_err(), "absolute path as filename must be rejected");
|
||||
}
|
||||
|
||||
/// On Windows, backslash is a path separator and must be rejected.
|
||||
#[cfg(windows)]
|
||||
#[test]
|
||||
fn test_new_rejects_backslash_path_separator_on_windows() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let result = RollingAppender::new(tmp.path(), "subdir\\app.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix);
|
||||
assert!(result.is_err(), "backslash path separator in filename must be rejected on Windows");
|
||||
}
|
||||
|
||||
// ── Basic writes ──────────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_write_stores_content() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let mut appender =
|
||||
RollingAppender::new(tmp.path(), "test.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix).unwrap();
|
||||
|
||||
appender.write_all(b"hello world\n").expect("write should succeed");
|
||||
appender.flush().expect("flush should succeed");
|
||||
|
||||
let content = fs::read_to_string(tmp.path().join("test.log")).unwrap();
|
||||
assert_eq!(content, "hello world\n");
|
||||
}
|
||||
|
||||
// ── Size-based rotation ────────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_size_rotation_creates_archive() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// Allow only 5 bytes before rotating.
|
||||
let mut appender =
|
||||
RollingAppender::new(tmp.path(), "app.log".to_string(), Rotation::Never, 5, FileMatchMode::Suffix).unwrap();
|
||||
|
||||
// First write: 5 bytes exactly — no rotation yet.
|
||||
appender.write_all(b"12345").expect("write should succeed");
|
||||
// Second write: would push past the limit — rotation should occur first.
|
||||
appender.write_all(b"abcde").expect("write after rotation should succeed");
|
||||
appender.flush().unwrap();
|
||||
|
||||
// There should now be 2 files: the active log + 1 archive.
|
||||
assert_eq!(count_files(tmp.path()), 2, "one rotation should have produced one archive");
|
||||
|
||||
// The active file should only contain the second write.
|
||||
let content = fs::read_to_string(tmp.path().join("app.log")).unwrap();
|
||||
assert_eq!(content, "abcde");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_size_rotations_produce_unique_archives() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
// Force a rotation on every write of 4+ bytes.
|
||||
let mut appender =
|
||||
RollingAppender::new(tmp.path(), "app.log".to_string(), Rotation::Never, 3, FileMatchMode::Suffix).unwrap();
|
||||
|
||||
for _ in 0..5 {
|
||||
appender.write_all(b"abcd").expect("write should succeed");
|
||||
}
|
||||
appender.flush().unwrap();
|
||||
|
||||
let file_count = count_files(tmp.path());
|
||||
// At least 5 archives (one per rotation) plus the active file.
|
||||
assert!(
|
||||
file_count >= 5,
|
||||
"each burst write should produce a distinct archive; got {file_count} files"
|
||||
);
|
||||
}
|
||||
|
||||
// ── Archive filename format ────────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_suffix_mode_archive_name() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let mut appender =
|
||||
RollingAppender::new(tmp.path(), "app.log".to_string(), Rotation::Never, 3, FileMatchMode::Suffix).unwrap();
|
||||
|
||||
appender.write_all(b"1234").expect("write should succeed");
|
||||
appender.flush().unwrap();
|
||||
|
||||
let archives: Vec<_> = fs::read_dir(tmp.path())
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.file_name().to_string_lossy().to_string())
|
||||
.filter(|n| n != "app.log")
|
||||
.collect();
|
||||
|
||||
assert_eq!(archives.len(), 1);
|
||||
// Suffix mode: "<timestamp>-<counter>.app.log"
|
||||
// Since timestamp contains digits and we use high precision, checking strictly is hard,
|
||||
// but it should definitely NOT be the old unix timestamp format (just digits).
|
||||
// It should contain "-" before "app.log" due to our new format.
|
||||
assert!(
|
||||
archives[0].ends_with(".app.log"),
|
||||
"archive should end with '.app.log' in Suffix mode; got '{}'",
|
||||
archives[0]
|
||||
);
|
||||
// Check for new format chars (YMD)
|
||||
// 20xx...
|
||||
assert!(
|
||||
archives[0].starts_with("20"),
|
||||
"archive should start with year (20xx); got '{}'",
|
||||
archives[0]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_prefix_mode_archive_name() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let mut appender =
|
||||
RollingAppender::new(tmp.path(), "app".to_string(), Rotation::Never, 3, FileMatchMode::Prefix).unwrap();
|
||||
|
||||
appender.write_all(b"1234").expect("write should succeed");
|
||||
appender.flush().unwrap();
|
||||
|
||||
let archives: Vec<_> = fs::read_dir(tmp.path())
|
||||
.unwrap()
|
||||
.filter_map(|e| e.ok())
|
||||
.map(|e| e.file_name().to_string_lossy().to_string())
|
||||
.filter(|n| n != "app")
|
||||
.collect();
|
||||
|
||||
assert_eq!(archives.len(), 1);
|
||||
// Prefix mode: "app.<timestamp>-<counter>"
|
||||
assert!(
|
||||
archives[0].starts_with("app.20"),
|
||||
"archive should start with 'app.20' in Prefix mode; got '{}'",
|
||||
archives[0]
|
||||
);
|
||||
}
|
||||
|
||||
// ── Restart with existing file ─────────────────────────────────────────────
|
||||
|
||||
#[test]
|
||||
fn test_restart_with_existing_file_reads_size() {
|
||||
let tmp = TempDir::new().unwrap();
|
||||
let log_path = tmp.path().join("app.log");
|
||||
fs::write(&log_path, b"existing content").unwrap();
|
||||
|
||||
let appender =
|
||||
RollingAppender::new(tmp.path(), "app.log".to_string(), Rotation::Daily, 0, FileMatchMode::Suffix).unwrap();
|
||||
|
||||
assert_eq!(
|
||||
appender.size,
|
||||
b"existing content".len() as u64,
|
||||
"size should reflect existing file content"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user