feat: delete orphaned components

Author: StefanFl

backend/application/background_tasks/periodic_tasks/core_tasks.py

@@ -4,7 +4,7 @@
 from application.background_tasks.services.task_base import so_periodic_task
 from application.commons import settings_static
 from application.core.services.housekeeping import (
-    delete_inactive_branches_and_set_flags,
+    housekeeping,
 )
 from application.core.services.risk_acceptance_expiry_task import (
     expire_risk_acceptances,
@@ -17,9 +17,9 @@
         hour=settings_static.branch_housekeeping_crontab_hour,
     )
 )
-@so_periodic_task("Branch housekeeping")
-def task_branch_housekeeping() -> str:
-    message = delete_inactive_branches_and_set_flags()
+@so_periodic_task("Housekeeping")
+def task_housekeeping() -> str:
+    message = housekeeping()
     return message
 
 

backend/application/background_tasks/services/registry.py

@@ -10,7 +10,7 @@
     ),
     "Import EPSS and cvss-bt": "application.background_tasks.periodic_tasks.epss_tasks.task_import_epss",
     "Import SPDX licenses": "application.background_tasks.periodic_tasks.license_tasks.task_spdx_license_import",
-    "Branch housekeeping": "application.background_tasks.periodic_tasks.core_tasks.task_branch_housekeeping",
+    "Housekeeping": "application.background_tasks.periodic_tasks.core_tasks.task_housekeeping",
     "Expire risk acceptances": "application.background_tasks.periodic_tasks.core_tasks.task_expire_risk_acceptances",
     "Calculate product metrics": (
         "application.background_tasks.periodic_tasks.metrics_tasks.task_calculate_product_metrics"

backend/application/core/services/housekeeping.py

@@ -1,16 +1,21 @@
 import logging
 import re
 from datetime import timedelta
+from itertools import batched
 
+from django.db.models import Exists, OuterRef
 from django.utils import timezone
 
 from application.commons.models import Settings
-from application.core.models import Branch, Observation, Product
+from application.core.models import Branch, Component, Observation, Product
+from application.licenses.models import License_Component
 
 logger = logging.getLogger("secobserve.core")
 
+BULK_BATCH_SIZE = 1000
 
-def delete_inactive_branches_and_set_flags() -> str:
+
+def housekeeping() -> str:
     num_products = 0
     num_deleted_branches = 0
     products = Product.objects.filter(is_product_group=False)
@@ -21,7 +26,12 @@ def delete_inactive_branches_and_set_flags() -> str:
             num_products += 1
         set_product_flags(product)
 
-    return f"Deleted {num_deleted_branches} inactive branches in {num_products} products."
+    num_deleted_components = delete_orphaned_components()
+
+    return (
+        f"Deleted {num_deleted_branches} inactive branches in {num_products} products."
+        f"\nDeleted {num_deleted_components} orphaned components."
+    )
 
 
 def delete_inactive_branches_for_product(product: Product) -> int:
@@ -129,3 +139,23 @@ def set_product_flags(product: Product) -> None:
         or has_potential_duplicates_before != product.has_potential_duplicates
     ):
         product.save()
+
+
+def delete_orphaned_components() -> int:
+    orphaned_components = Component.objects.filter(
+        ~Exists(Observation.objects.filter(origin_component=OuterRef("pk"))),
+        ~Exists(License_Component.objects.filter(component=OuterRef("pk"))),
+    )
+    orphaned_component_ids = list(orphaned_components.values_list("pk", flat=True))
+
+    num_deleted_components = 0
+    for component_ids_batch in batched(orphaned_component_ids, BULK_BATCH_SIZE):
+        deleted_objects = orphaned_components.filter(pk__in=component_ids_batch).delete()
+        num_deleted_components += deleted_objects[1].get("core.Component", 0)
+
+    if num_deleted_components:
+        logger.info(  # pylint: disable=logging-fstring-interpolation
+            f"Deleted {num_deleted_components} orphaned components"
+        )
+
+    return num_deleted_components

frontend/src/commons/settings/SettingsEdit.tsx

@@ -559,7 +559,7 @@ const SettingsEdit = () => {
                                 </FormDataConsumer>
                                 <NumberInput
                                     source="branch_housekeeping_crontab_hour"
-                                    label="Branch housekeeping crontab (hour)"
+                                    label="Housekeeping crontab (hour)"
                                     min={0}
                                     step={1}
                                     validate={validate_0_23}
@@ -625,7 +625,7 @@ const SettingsEdit = () => {
                                 </FormDataConsumer>
                                 <NumberInput
                                     source="branch_housekeeping_crontab_minute"
-                                    label="Branch housekeeping crontab (minute)"
+                                    label="Housekeeping crontab (minute)"
                                     min={0}
                                     step={1}
                                     validate={validate_0_59}

frontend/src/commons/settings/SettingsShow.tsx

@@ -359,7 +359,7 @@ const SettingsShowComponent = () => {
                                             <NumberField source="license_import_crontab_hour" />
                                         </Labeled>
                                     )}
-                                    <Labeled label="Branch housekeeping crontab (hour/UTC)">
+                                    <Labeled label="Housekeeping crontab (hour/UTC)">
                                         <NumberField source="branch_housekeeping_crontab_hour" />
                                     </Labeled>
                                     <Labeled label="EPSS and exploit import crontab (hour/UTC)">
@@ -386,7 +386,7 @@ const SettingsShowComponent = () => {
                                             <NumberField source="license_import_crontab_minute" />
                                         </Labeled>
                                     )}
-                                    <Labeled label="Branch housekeeping crontab (minute)">
+                                    <Labeled label="Housekeeping crontab (minute)">
                                         <NumberField source="branch_housekeeping_crontab_minute" />
                                     </Labeled>
                                     <Labeled label="EPSS and exploit import crontab (minutes)">