GKaszewski/movies-diary
1
0
Fork 0
Code Issues 9 Pull Requests Actions Packages Projects Releases Wiki Activity

purge: skip broadcasts of deleted authors, purge outbox log, guards, 410 inbox test

Browse Source
This commit is contained in:
Gabriel Kaszewski
2026-10-26 02:39:06 +01:00
parent 51a70b5260
commit 80dbdec107
9 changed files with 318 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
crates/adapters/sqlite/src/lib.rs
View File

@@ -8,15 +8,15 @@ mod shared;
pub use event_queue::{DbEventQueueConfig, SqliteEventQueue};
use repositories::SqliteUserSettingsRepository;
pub use repositories::{
PURGED_USER_TABLES, RETAINED_USER_TABLES, SqliteAccountPurger, SqliteApContentQuery,
SqliteDiaryRepository, SqliteGoalRepository, SqliteImageRefAdapter,
PAYLOAD_KEYED_USER_COLUMNS, PURGED_USER_TABLES, RETAINED_USER_TABLES, SqliteAccountPurger,
SqliteApContentQuery, SqliteDiaryRepository, SqliteGoalRepository, SqliteImageRefAdapter,
SqliteImportProfileRepository, SqliteImportSessionRepository, SqliteInsightsQuery,
SqliteMovieDeduplicator, SqliteMovieProfileRepository, SqliteMovieRepository,
SqlitePersonAdapter, SqliteProcessedEventLedger, SqliteProfileFieldsRepository,
SqliteRefreshSessionAdapter, SqliteRemoteGoalRepository, SqliteReviewRepository,
SqliteSocialRepository, SqliteStatsRepository, SqliteUserRepository,
SqliteWatchEventRepository, SqliteWatchlistRepository, SqliteWebhookTokenRepository,
SqliteWrapUpRepository, SqliteWrapUpStatsQuery, URL_KEYED_USER_TABLES,
SqliteWrapUpRepository, SqliteWrapUpStatsQuery, URL_KEYED_USER_COLUMNS,
create_federated_profile_query, create_image_ref, create_person_adapter, create_search_adapter,
};

48
crates/adapters/sqlite/src/repositories/account_purge/mod.rs
View File

@@ -34,9 +34,19 @@ pub const PURGED_USER_TABLES: &[(&str, &str)] = &[
/// the keypair is needed to sign queued federation deliveries.
pub const RETAINED_USER_TABLES: &[&str] = &["ap_local_actors"];
/// Tables with no user column whose rows hang off the user's content or actor
/// URL; the purge clears them by URL.
pub const URL_KEYED_USER_TABLES: &[&str] = &["ap_announces", "ap_followers", "ap_following"];
/// Columns that identify the user by actor or content URL rather than by id.
/// The purge matches rows on them; the guard test asserts each still exists so
/// a rename cannot silently stop the purge from clearing them.
pub const URL_KEYED_USER_COLUMNS: &[(&str, &str)] = &[
("ap_announces", "object_url"),
("ap_announces", "actor_url"),
("ap_followers", "remote_actor_url"),
("ap_following", "remote_actor_url"),
];
/// Columns holding serialized events and activities that carry the user's id
/// (the outbox log keeps finished rows, including authored activity JSON).
pub const PAYLOAD_KEYED_USER_COLUMNS: &[(&str, &str)] = &[("event_queue", "payload")];
pub struct SqliteAccountPurger {
pool: SqlitePool,
@@ -116,6 +126,22 @@ impl AccountPurger for SqliteAccountPurger {
.await
.map_err(map_sqlx_error)?;
// The outbox log: events and signed activities authored as this user.
// The `UserDeleted` row (which triggered this run) and the pending
// `Delete` deliveries must stay; both go with the tombstone.
sqlx::query(
"DELETE FROM event_queue
WHERE instr(payload, ?1) > 0
AND event_type != 'UserDeleted'
AND NOT (event_type = 'FederationDeliveryRequested'
AND status IN ('pending', 'processing')
AND instr(payload, '\"type\":\"Delete\"') > 0)",
)
.bind(&id)
.execute(&mut *tx)
.await
.map_err(map_sqlx_error)?;
// Other accounts' follows of this one.
for table in ["ap_followers", "ap_following"] {
sqlx::query(&format!("DELETE FROM {table} WHERE remote_actor_url = ?"))
@@ -158,6 +184,22 @@ impl AccountPurger for SqliteAccountPurger {
) -> Result<u64, DomainError> {
let cutoff = cutoff.format(TIMESTAMP_FORMAT).to_string();
let mut tx = self.pool.begin().await.map_err(map_sqlx_error)?;
let expiring: Vec<String> = sqlx::query_scalar(
"SELECT id FROM users WHERE purged_at IS NOT NULL AND purged_at < ?",
)
.bind(&cutoff)
.fetch_all(&mut *tx)
.await
.map_err(map_sqlx_error)?;
for id in &expiring {
// What the purge had to leave: the `UserDeleted` row and the
// `Delete` deliveries.
sqlx::query("DELETE FROM event_queue WHERE instr(payload, ?) > 0")
.bind(id)
.execute(&mut *tx)
.await
.map_err(map_sqlx_error)?;
}
for table in RETAINED_USER_TABLES {
sqlx::query(&format!(
"DELETE FROM {table} WHERE user_id IN (

3
crates/adapters/sqlite/src/repositories/mod.rs
View File

@@ -25,7 +25,8 @@ mod watchlist;
mod wrapup;
pub use account_purge::{
PURGED_USER_TABLES, RETAINED_USER_TABLES, SqliteAccountPurger, URL_KEYED_USER_TABLES,
PAYLOAD_KEYED_USER_COLUMNS, PURGED_USER_TABLES, RETAINED_USER_TABLES, SqliteAccountPurger,
URL_KEYED_USER_COLUMNS,
};
pub use ap_content::SqliteApContentQuery;
pub use diary::SqliteDiaryRepository;

98
crates/adapters/sqlite/tests/account_purge_test.rs
View File

@@ -14,9 +14,9 @@ use domain::ports::{
use domain::testing::{test_goal, test_movie, test_review, test_user, test_watched_at};
use domain::value_objects::{FollowStatus, InstanceIdentity, UserId};
use sqlite::{
PURGED_USER_TABLES, RETAINED_USER_TABLES, SqliteAccountPurger, SqliteProfileFieldsRepository,
SqliteSocialRepository, SqliteWatchEventRepository, SqliteWebhookTokenRepository,
URL_KEYED_USER_TABLES,
PAYLOAD_KEYED_USER_COLUMNS, PURGED_USER_TABLES, RETAINED_USER_TABLES, SqliteAccountPurger,
SqliteProfileFieldsRepository, SqliteSocialRepository, SqliteWatchEventRepository,
SqliteWebhookTokenRepository, URL_KEYED_USER_COLUMNS,
};
use sqlx::SqlitePool;
use sqlx::sqlite::{SqliteConnectOptions, SqlitePoolOptions};
@@ -266,6 +266,45 @@ async fn seed(pool: &SqlitePool, parts: &sqlite::SqliteWireOutput) -> Seeded {
)
.await;
}
for (event_type, status, payload) in [
(
"ReviewLogged",
"done",
format!(r#"{{"user_id":"{a_str}","comment":"secret"}}"#),
),
(
"FederationDeliveryRequested",
"done",
format!(
r#"{{"signing_actor_id":"{a_str}","activity_json":{{"type":"Create","content":"secret"}}}}"#
),
),
(
"FederationDeliveryRequested",
"pending",
format!(
r#"{{"signing_actor_id":"{a_str}","activity_json":{{"type":"Delete","id":"x"}}}}"#
),
),
(
"UserDeleted",
"processing",
format!(r#"{{"user_id":"{a_str}"}}"#),
),
(
"ReviewLogged",
"done",
format!(r#"{{"user_id":"{b_str}"}}"#),
),
] {
sqlx::query("INSERT INTO event_queue (event_type, payload, status) VALUES (?, ?, ?)")
.bind(event_type)
.bind(payload)
.bind(status)
.execute(pool)
.await
.unwrap();
}
Seeded {
user: a,
other: b,
@@ -332,6 +371,24 @@ async fn purge_removes_every_user_owned_row() {
.unwrap();
assert_eq!(announces, vec!["an3"], "only bob's announce survives");
let queue: Vec<(String, String)> =
sqlx::query_as("SELECT event_type, status FROM event_queue ORDER BY id")
.fetch_all(&pool)
.await
.unwrap();
assert_eq!(
queue,
vec![
(
"FederationDeliveryRequested".to_string(),
"pending".to_string()
),
("UserDeleted".to_string(), "processing".to_string()),
("ReviewLogged".to_string(), "done".to_string()),
],
"the user's authored events and activities are gone; the pending Delete, the trigger and bob's event stay"
);
// The other account is untouched.
for (table, column) in PURGED_USER_TABLES {
assert!(
@@ -384,17 +441,36 @@ async fn the_purge_table_lists_cover_every_user_referencing_column_in_the_schema
"{t} has a foreign key to users but is not in the purge lists"
);
}
// URL-keyed tables must exist, so a rename cannot silently drop them.
for t in URL_KEYED_USER_TABLES {
let n: i64 = count(
&pool,
&format!("SELECT COUNT(*) FROM sqlite_master WHERE type='table' AND name='{t}'"),
)
.await;
assert_eq!(n, 1, "{t} is listed as URL-keyed but missing");
// Columns the purge matches by URL or payload must still exist, so a
// rename fails here instead of silently leaving rows behind.
for (table, column) in URL_KEYED_USER_COLUMNS
.iter()
.chain(PAYLOAD_KEYED_USER_COLUMNS)
{
let n: i64 = sqlx::query_scalar("SELECT COUNT(*) FROM pragma_table_info(?) WHERE name = ?")
.bind(table)
.bind(column)
.fetch_one(&pool)
.await
.unwrap();
assert_eq!(n, 1, "{table}.{column} is matched by the purge but missing");
}
}
/// `ap_activities` (k-ap's inbound dedup) holds only remote activity ids and a
/// timestamp: nothing in it is attributable to a local user. If it ever grows a
/// column, the purge must start clearing it.
#[tokio::test]
async fn ap_activities_holds_nothing_attributable_to_a_local_user() {
let pool = migrated_pool().await;
let columns: Vec<String> =
sqlx::query_scalar("SELECT name FROM pragma_table_info('ap_activities') ORDER BY name")
.fetch_all(&pool)
.await
.unwrap();
assert_eq!(columns, vec!["id", "processed_at"]);
}
#[tokio::test]
async fn purge_keeps_movies_and_metadata() {
let pool = migrated_pool().await;