sdgoij/stalwart
1
0
Fork 0
mirror of https://github.com/stalwartlabs/stalwart.git synced 2026-03-17 14:34:03 +00:00
Code Issues Packages Projects Releases Wiki Activity

PostgreSQL: Limit document size to 650kb

Browse Source
This commit is contained in:
mdecimus
2025-12-27 16:55:25 +01:00
parent e03629d1d2
commit 5e9d78567d
2 changed files with 46 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
crates/store/src/backend/postgres/search.rs
View File

@@ -318,9 +318,8 @@ impl ToSql for SearchValue {
SearchValue::Text { value, .. } => {
// Truncate large text fields to avoid Postgres errors (see https://www.postgresql.org/docs/current/textsearch-limitations.html)
if value.len() > 1_048_574 {
let pos = value.floor_char_boundary(1_048_574);
(&value[..pos]).to_sql(ty, out)
if value.len() > 650_000 {
(&value[..value.floor_char_boundary(650_000)]).to_sql(ty, out)
} else {
value.to_sql(ty, out)
}
@@ -356,9 +355,8 @@ impl ToSql for SearchValue {
SearchValue::Text { value, .. } => {
// Truncate large text fields to avoid Postgres errors (see https://www.postgresql.org/docs/current/textsearch-limitations.html)
if value.len() > 1_048_574 {
let pos = value.floor_char_boundary(1_048_574);
(&value[..pos]).to_sql_checked(ty, out)
if value.len() > 650_000 {
(&value[..value.floor_char_boundary(650_000)]).to_sql_checked(ty, out)
} else {
value.to_sql_checked(ty, out)
}

42
tests/src/store/query.rs
View File

@@ -15,6 +15,7 @@ use std::{
use store::{
SearchStore,
ahash::AHashMap,
rand::{self, Rng, distr::Alphanumeric},
roaring::RoaringBitmap,
search::{
EmailSearchField, IndexDocument, SearchComparator, SearchField, SearchFilter,
@@ -120,6 +121,47 @@ pub async fn test(store: SearchStore, do_insert: bool) {
println!("Running global id filtering tests...");
test_global(store.clone()).await;
// Large document insert test
println!("Running large document insert tests...");
let mut large_text = String::with_capacity(20 * 1024 * 1024);
while large_text.len() < 20 * 1024 * 1024 {
let word = rand::rng()
.sample_iter(&Alphanumeric)
.take(rand::rng().random_range(3..10))
.map(char::from)
.collect::<String>();
large_text.push_str(&word);
large_text.push(' ');
}
let mut document = IndexDocument::new(SearchIndex::Email)
.with_account_id(1)
.with_document_id(1);
for field in [
EmailSearchField::From,
EmailSearchField::To,
EmailSearchField::Cc,
EmailSearchField::Bcc,
EmailSearchField::Subject,
] {
document.index_text(field, &large_text[..10 * 1024], Language::English);
}
for field in [EmailSearchField::Body, EmailSearchField::Attachment] {
document.index_text(field, &large_text, Language::English);
}
for field in [
EmailSearchField::ReceivedAt,
EmailSearchField::SentAt,
EmailSearchField::Size,
] {
document.index_unsigned(field, rand::rng().random_range(100u64..1_000_000u64));
}
store.index(vec![document]).await.unwrap();
// Refresh
if let SearchStore::ElasticSearch(store) = &store {
store.refresh_index(SearchIndex::Email).await.unwrap();
}
println!("Running account filtering tests...");
let filter_ids = std::env::var("QUICK_TEST").is_ok().then(|| {
let mut ids = AHashSet::new();
for &id in ALL_IDS {