From 0ff9116d5740cec6dd367c6366ed0daf312c5863 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Thu, 15 Jan 2026 13:44:08 -0500 Subject: [PATCH 1/9] Delete background jobs on org delete --- backend/btrixcloud/orgs.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index 6c9ab2f313..daf5e5c747 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -228,6 +228,7 @@ def __init__( self.pages_db = mdb["pages"] self.version_db = mdb["version"] self.invites_db = mdb["invites"] + self.jobs_db = mdb["jobs"] self.router = None self.org_viewer_dep = None @@ -1546,6 +1547,9 @@ async def delete_org_and_data( # Delete invites await self.invites_db.delete_many({"oid": org.id}) + # Delete background jobs + await self.jobs_db.delete_many({"oid": org.id}) + # Delete org await self.orgs.delete_one({"_id": org.id}) From e11434af3e177b0ec530565e897ed07a7900740b Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Tue, 20 Jan 2026 13:21:48 -0500 Subject: [PATCH 2/9] Add migration to delete bg jobs for deleted orgs from db --- backend/btrixcloud/db.py | 2 +- ...gration_0057_deleted_org_bg_job_cleanup.py | 48 +++++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) create mode 100644 backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py diff --git a/backend/btrixcloud/db.py b/backend/btrixcloud/db.py index 5f5e0f7312..ce18e12b5e 100644 --- a/backend/btrixcloud/db.py +++ b/backend/btrixcloud/db.py @@ -44,7 +44,7 @@ ) = object -CURR_DB_VERSION = "0056" +CURR_DB_VERSION = "0057" MIN_DB_VERSION = 7.0 diff --git a/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py b/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py new file mode 100644 index 0000000000..2e2219f633 --- /dev/null +++ b/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py @@ -0,0 +1,48 @@ +""" +Migration 0057 - Remove background jobs for deleted orgs from db +""" + +from motor.motor_asyncio import AsyncIOMotorDatabase + +from btrixcloud.migrations import BaseMigration + +MIGRATION_VERSION = "0057" + + +class Migration(BaseMigration): + """Migration class.""" + + # pylint: disable=unused-argument + def __init__(self, mdb: AsyncIOMotorDatabase, **kwargs): + super().__init__(mdb, migration_version=MIGRATION_VERSION) + + async def migrate_up(self): + """Perform migration up. + + Delete background jobs from deleted orgs from the database. + """ + # pylint: disable=duplicate-code + jobs_mdb = self.mdb["jobs"] + orgs_mdb = self.mdb["organizations"] + + job_orgs_to_delete: list[UUID] = [] + + job_oids = await jobs_mdb.distinct("oid", {}) + + for oid in job_oids: + res = await orgs_mdb.find_one({"_id": oid}) + if res is None: + job_orgs_to_delete.append(oid) + + if job_orgs_to_delete: + del_count = len(job_orgs_to_delete) + print( + f"Deleting background jobs for {del_count} deleted orgs", + flush=True, + ) + + try: + await jobs_mdb.delete_many({"oid": {"$in": job_orgs_to_delete}}) + # pylint: disable=broad-exception-caught + except Exception as err: + print(f"Error deleting jobs from deleted orgs: {err}", flush=True) From f92485d9ae6b96b9202f7cae737333de3af92b94 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Tue, 20 Jan 2026 15:31:45 -0500 Subject: [PATCH 3/9] Add UUID import --- .../migrations/migration_0057_deleted_org_bg_job_cleanup.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py b/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py index 2e2219f633..f14a9411fb 100644 --- a/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py +++ b/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py @@ -2,6 +2,8 @@ Migration 0057 - Remove background jobs for deleted orgs from db """ +from uuid import UUID + from motor.motor_asyncio import AsyncIOMotorDatabase from btrixcloud.migrations import BaseMigration From 7af93466a9e573cfaa579b065d323a874857c1d3 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Wed, 25 Mar 2026 12:32:14 -0400 Subject: [PATCH 4/9] Modify org deletion to be safer - Only delete k8s resources in operator after job completes to prevent killing job before it has a chance to delete everything - Delete all background jobs from database except the delete org job so there's a record of the org having existed and so that tests can ensure deletion is successful - Ensure that all calls of delete_org_and_data method launch a background job for consistency - Don't delete org deletion background jobs from db in migration --- backend/btrixcloud/background_jobs.py | 7 ++++++- ...gration_0057_deleted_org_bg_job_cleanup.py | 8 +++++++- backend/btrixcloud/orgs.py | 20 +++++++++++++------ backend/btrixcloud/subs.py | 2 +- 4 files changed, 28 insertions(+), 9 deletions(-) diff --git a/backend/btrixcloud/background_jobs.py b/backend/btrixcloud/background_jobs.py index 15bb450f7b..bc0d9b298c 100644 --- a/backend/btrixcloud/background_jobs.py +++ b/backend/btrixcloud/background_jobs.py @@ -519,8 +519,13 @@ async def job_finished( await self._send_bg_job_failure_email(cleanup_job, finished) return + # If org has been successfully deleted in job, delete k8s resources + # associated with this org now that no jobs are running + if job_type == BgJobType.DELETE_ORG and oid and success: + await self.crawl_manager.delete_all_k8s_resources_for_org(str(oid)) + job = await self.get_background_job(job_id) - if job.finished: + if not job or job.finished: return if job.type != job_type: diff --git a/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py b/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py index f14a9411fb..fda3a96888 100644 --- a/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py +++ b/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py @@ -7,6 +7,7 @@ from motor.motor_asyncio import AsyncIOMotorDatabase from btrixcloud.migrations import BaseMigration +from btrixcloud.models import BgJobType MIGRATION_VERSION = "0057" @@ -44,7 +45,12 @@ async def migrate_up(self): ) try: - await jobs_mdb.delete_many({"oid": {"$in": job_orgs_to_delete}}) + await jobs_mdb.delete_many( + { + "oid": {"$in": job_orgs_to_delete}, + "type": {"$ne": BgJobType.DELETE_ORG}, + } + ) # pylint: disable=broad-exception-caught except Exception as err: print(f"Error deleting jobs from deleted orgs: {err}", flush=True) diff --git a/backend/btrixcloud/orgs.py b/backend/btrixcloud/orgs.py index daf5e5c747..c887566e3f 100644 --- a/backend/btrixcloud/orgs.py +++ b/backend/btrixcloud/orgs.py @@ -101,6 +101,7 @@ OrgPublicProfileUpdate, MAX_BROWSER_WINDOWS, MAX_CRAWL_SCALE, + BgJobType, ) from .pagination import DEFAULT_PAGE_SIZE, paginated_format from .utils import ( @@ -1495,7 +1496,13 @@ async def import_org( async def delete_org_and_data( self, org: Organization, user_manager: UserManager ) -> None: - """Delete org and all of its associated data.""" + """Delete org and all of its associated data. + + This method should only be run in a background job. The operator + will delete associated k8s resources for this org when the job + successfully completes to prevent deleting this job before it + completes. + """ print(f"Deleting org: {org.slug} {org.name} {org.id}") # Delete archived items @@ -1547,14 +1554,15 @@ async def delete_org_and_data( # Delete invites await self.invites_db.delete_many({"oid": org.id}) - # Delete background jobs - await self.jobs_db.delete_many({"oid": org.id}) - # Delete org await self.orgs.delete_one({"_id": org.id}) - # Delete related k8s objects - await self.crawl_manager.delete_all_k8s_resources_for_org(str(org.id)) + # Delete all background jobs except this one from database, + # so that we are left with some record of the org having + # existed and successfully deleted + await self.orgs.delete_one( + {"_id": org.id, "type": {"$ne": BgJobType.DELETE_ORG}} + ) async def recalculate_storage(self, org: Organization) -> dict[str, bool]: """Recalculate org storage use""" diff --git a/backend/btrixcloud/subs.py b/backend/btrixcloud/subs.py index 970ea83bef..a66e3e5a17 100644 --- a/backend/btrixcloud/subs.py +++ b/backend/btrixcloud/subs.py @@ -198,7 +198,7 @@ async def cancel_subscription(self, cancel: SubscriptionCancel) -> dict[str, boo ) if not org.subscription.readOnlyOnCancel: - await self.org_ops.delete_org_and_data(org, self.user_manager) + await self.org_ops.background_job_ops.create_delete_org_job(org) deleted = True await self.add_sub_event("cancel", cancel, org.id) From 079e7a7b9defb0ee373fe91024931cee15b2b1d5 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Wed, 25 Mar 2026 16:10:52 -0400 Subject: [PATCH 5/9] Fix cancel subscription test --- backend/test/test_org_subs.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/backend/test/test_org_subs.py b/backend/test/test_org_subs.py index 916d4c6adf..a645db95ae 100644 --- a/backend/test/test_org_subs.py +++ b/backend/test/test_org_subs.py @@ -10,6 +10,8 @@ new_user_invite_token = None existing_user_invite_token = None +MAX_ATTEMPTS = 24 + VALID_PASSWORD = "ValidPassW0rd!" invite_email = "test-User@EXample.com" @@ -357,9 +359,20 @@ def test_cancel_sub_and_delete_org(admin_auth_headers): assert r.status_code == 200 assert r.json() == {"canceled": True, "deleted": True} - r = requests.get(f"{API_PREFIX}/orgs/{new_subs_oid}", headers=admin_auth_headers) - assert r.status_code == 404 - assert r.json()["detail"] == "org_not_found" + # Wait for org to be deleted + count = 0 + while count < MAX_ATTEMPTS: + r = requests.get( + f"{API_PREFIX}/orgs/{new_subs_oid}", headers=admin_auth_headers + ) + if r.status_code == 404: + assert r.json().get("detail") == "org_not_found" + + if count + 1 == MAX_ATTEMPTS: + assert False + + time.sleep(10) + count += 1 def test_cancel_sub_and_no_delete_org(admin_auth_headers): From 41c2b4ff84e3bb091e452849857404f41a7d4741 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Wed, 25 Mar 2026 18:49:47 -0400 Subject: [PATCH 6/9] Add missing test import --- backend/test/test_org_subs.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/test/test_org_subs.py b/backend/test/test_org_subs.py index a645db95ae..22fcd65221 100644 --- a/backend/test/test_org_subs.py +++ b/backend/test/test_org_subs.py @@ -1,8 +1,8 @@ import requests - -from .conftest import API_PREFIX +import time from uuid import uuid4 +from .conftest import API_PREFIX new_subs_oid = None new_subs_oid_2 = None From 966886df79fcad7350b7804a38bb1a4c2bd50fca Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Thu, 26 Mar 2026 01:14:12 -0400 Subject: [PATCH 7/9] Add missing break --- backend/test/test_org_subs.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/test/test_org_subs.py b/backend/test/test_org_subs.py index 22fcd65221..9314ec4c2b 100644 --- a/backend/test/test_org_subs.py +++ b/backend/test/test_org_subs.py @@ -367,6 +367,7 @@ def test_cancel_sub_and_delete_org(admin_auth_headers): ) if r.status_code == 404: assert r.json().get("detail") == "org_not_found" + break if count + 1 == MAX_ATTEMPTS: assert False From f2816d780ea0eef27d67064a614a1e1039302c82 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Thu, 9 Apr 2026 16:28:20 -0400 Subject: [PATCH 8/9] Log how many background jobs are deleted in migration --- .../migrations/migration_0057_deleted_org_bg_job_cleanup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py b/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py index fda3a96888..6c0bb860fe 100644 --- a/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py +++ b/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py @@ -45,12 +45,13 @@ async def migrate_up(self): ) try: - await jobs_mdb.delete_many( + res = await jobs_mdb.delete_many( { "oid": {"$in": job_orgs_to_delete}, "type": {"$ne": BgJobType.DELETE_ORG}, } ) + print(f"Deleted {res.deleted_count} jobs from database", flush=True) # pylint: disable=broad-exception-caught except Exception as err: print(f"Error deleting jobs from deleted orgs: {err}", flush=True) From a063d60fb4a1f480108d98f583b32c425c957590 Mon Sep 17 00:00:00 2001 From: Tessa Walsh Date: Thu, 9 Apr 2026 16:34:34 -0400 Subject: [PATCH 9/9] Add comment --- .../migrations/migration_0057_deleted_org_bg_job_cleanup.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py b/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py index 6c0bb860fe..6bb9a2b74c 100644 --- a/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py +++ b/backend/btrixcloud/migrations/migration_0057_deleted_org_bg_job_cleanup.py @@ -48,6 +48,9 @@ async def migrate_up(self): res = await jobs_mdb.delete_many( { "oid": {"$in": job_orgs_to_delete}, + # Maintain consistency with behavior moving forward, to + # retain only the one org deletion background job from + # deleted orgs "type": {"$ne": BgJobType.DELETE_ORG}, } )