From dd91685f76623d70d9be732be680599ca9097067 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 17 Aug 2026 12:46:47 -0700
Subject: [PATCH 01/78] Bound compare_body input to prevent quadratic DeepDiff
 stall

HttpCompare.compare_body ran DeepDiff with ignore_order=True on line-lists.
When two bodies are highly similar but differ on many lines (the exact shape
wildcard detection produces on large catch-all hosts), DeepDiff's pairing is
quadratic in the differing-line count and holds the GIL, stalling entire scans
for tens of minutes.

Short-circuit to 'different' when the multiset difference of two line-lists
exceeds web.http_compare_max_differing_lines (default 500) before invoking
DeepDiff. Caps worst-case comparison at a couple seconds. Dict inputs
(XML/JSON) keep the full diff path unchanged.

Fixes #3339
---
 bbot/core/helpers/diff.py         | 10 ++++++++++
 bbot/defaults.yml                 |  1 +
 bbot/test/test_step_1/test_web.py | 11 +++++++++++
 3 files changed, 22 insertions(+)

diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py
index 126f122e68..08a5ef04e2 100644
--- a/bbot/core/helpers/diff.py
+++ b/bbot/core/helpers/diff.py
@@ -1,5 +1,6 @@
 import logging
 import xmltodict
+from collections import Counter
 from deepdiff import DeepDiff
 from contextlib import suppress
 from xml.parsers.expat import ExpatError
@@ -99,6 +100,7 @@ def __init__(
         self.headers = headers
         self.cookies = cookies
         self.timeout = 10
+        self.max_differing_lines = self.parent_helper.web_config.get("http_compare_max_differing_lines", 500)
         # Optional async callback fired once with baseline_1 after the baseline is established.
         self.on_baseline_ready = on_baseline_ready
 
@@ -239,6 +241,14 @@ def compare_body(self, content_1, content_2):
         if content_1 == content_2:
             return True
 
+        if isinstance(content_1, list) and isinstance(content_2, list):
+            counts_1 = Counter(content_1)
+            counts_2 = Counter(content_2)
+            differing = counts_1 - counts_2
+            differing.update(counts_2 - counts_1)
+            if sum(differing.values()) > self.max_differing_lines:
+                return False
+
         ddiff = DeepDiff(
             content_1,
             content_2,
diff --git a/bbot/defaults.yml b/bbot/defaults.yml
index 3a3d53a55c..a3711f6d39 100644
--- a/bbot/defaults.yml
+++ b/bbot/defaults.yml
@@ -130,6 +130,7 @@ web:
   debug: false
   # Maximum number of HTTP redirects to follow
   http_max_redirects: 5
+  http_compare_max_differing_lines: 500
   # Whether to verify SSL certificates for target-directed traffic (probes, crawls, etc.)
   ssl_verify_target: false
   # Whether to verify SSL certificates for non-target traffic (APIs, wordlist downloads, etc.)
diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index 530cc66b97..ccaea047ed 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -1,4 +1,5 @@
 import re
+import time
 
 from blasthttp import HTTPStatusError
 
@@ -423,6 +424,16 @@ async def test_web_http_compare(blasthttp_mock, bbot_scanner):
     for mode in ("getparam", "header", "cookie"):
         assert await compare_helper.canary_check("http://www.example.com", mode=mode) is True
 
+    assert compare_helper.compare_body(["a", "b", "c"], ["a", "b", "c"]) is True
+    assert compare_helper.compare_body(["a", "b", "c"], ["a", "b", "x"]) is False
+
+    compare_helper.max_differing_lines = 10
+    base = [f"line {i}" for i in range(2000)]
+    similar = [f"line {i} X" if i < 50 else f"line {i}" for i in range(2000)]
+    start = time.monotonic()
+    assert compare_helper.compare_body(base, similar) is False
+    assert (time.monotonic() - start) < 5
+
     await scan._cleanup()
 
 

From 6902306621bb7ad2eaf328adff3668b9e87fd1c2 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 17 Aug 2026 13:41:20 -0700
Subject: [PATCH 02/78] Register http_compare_max_differing_lines in WebConfig
 schema

defaults.yml added the key; the strict pydantic WebConfig model must
accept it or test_defaults_yml_validates_against_schema fails.
---
 bbot/core/config/models.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/bbot/core/config/models.py b/bbot/core/config/models.py
index 3f3e85105d..1652f0c13b 100644
--- a/bbot/core/config/models.py
+++ b/bbot/core/config/models.py
@@ -294,6 +294,7 @@ class WebConfig(BaseModel):
     max_sleep_interval_429: Optional[int] = Field(default=None, alias="429_max_sleep_interval")
     debug: Optional[bool] = None
     http_max_redirects: Optional[int] = None
+    http_compare_max_differing_lines: Optional[int] = None
     ssl_verify_target: Optional[bool] = None
     ssl_verify_infrastructure: Optional[bool] = None
 

From 101bae53eec3d67f74daba67d64996dc6b42d8cb Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 18 Aug 2026 06:39:11 -0700
Subject: [PATCH 03/78] Broaden compare_body tests: dict path, threshold
 boundary, config knob

- dict inputs still take the DeepDiff path (guard must not short-circuit them)
- reordered identical line-lists stay equal (multiset is order-insensitive)
- at-threshold vs over-threshold boundary around max_differing_lines
- http_compare_max_differing_lines config value reaches the helper
---
 bbot/test/test_step_1/test_web.py | 26 ++++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index ccaea047ed..fe6c0a687c 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -426,14 +426,36 @@ async def test_web_http_compare(blasthttp_mock, bbot_scanner):
 
     assert compare_helper.compare_body(["a", "b", "c"], ["a", "b", "c"]) is True
     assert compare_helper.compare_body(["a", "b", "c"], ["a", "b", "x"]) is False
+    assert compare_helper.compare_body(["a", "b", "c"], ["c", "b", "a"]) is True
+    assert compare_helper.compare_body({"a": 1}, {"a": 2}) is False
+    assert compare_helper.compare_body({"a": 1}, {"a": 1}) is True
 
     compare_helper.max_differing_lines = 10
-    base = [f"line {i}" for i in range(2000)]
-    similar = [f"line {i} X" if i < 50 else f"line {i}" for i in range(2000)]
+    shared = [f"line {i}" for i in range(100)]
+    at_threshold = compare_helper.compare_body(
+        shared + [f"a{i}" for i in range(5)],
+        shared + [f"b{i}" for i in range(5)],
+    )
+    over_threshold = compare_helper.compare_body(
+        shared + [f"a{i}" for i in range(6)],
+        shared + [f"b{i}" for i in range(6)],
+    )
+    assert at_threshold is False
+    assert over_threshold is False
+
+    compare_helper.max_differing_lines = 500
+    base = [f"line {i}" for i in range(20000)]
+    similar = [f"line {i} X" if i < 5000 else f"line {i}" for i in range(20000)]
     start = time.monotonic()
     assert compare_helper.compare_body(base, similar) is False
     assert (time.monotonic() - start) < 5
 
+    config_scan = bbot_scanner(config={"web": {"http_compare_max_differing_lines": 42}})
+    await config_scan._prep()
+    config_helper = config_scan.helpers.http_compare("http://www.example.com")
+    assert config_helper.max_differing_lines == 42
+    await config_scan._cleanup()
+
     await scan._cleanup()
 
 

From 42164af3870c3265cc5f01b71cd14f548ba263d6 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 09:11:42 -0700
Subject: [PATCH 04/78] test(diff): pin compare_body regressions the threshold
 guard introduced

Four focused tests, all failing against the current guard:

filtered_lines_not_counted asserts a subject differing only in lines
the baseline already marked dynamic still compares equal. The guard
counts those lines before DeepDiff excludes them, so it returns a
false "changed" on any page with many nonces.

bounds_dict_bodies asserts an XML/JSON body settles in bounded time.
xmltodict.parse succeeds on ordinary HTML, so those bodies are dicts
and skip the list-only guard into the unbounded quadratic path.

threshold_boundary makes the at-threshold case one DeepDiff calls a
match, so the assertion can distinguish the guard firing from the
lists simply differing.

null_threshold_config covers an explicit null, which resolves to None
and raises TypeError on the comparison.

Replaces the previous boundary assertions, which expected False on
both sides and passed with the feature deleted.
---
 bbot/test/test_step_1/test_web.py | 105 ++++++++++++++++++++++++++----
 1 file changed, 92 insertions(+), 13 deletions(-)

diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index fe6c0a687c..ac7a1330cc 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -2,6 +2,7 @@
 import time
 
 from blasthttp import HTTPStatusError
+from deepdiff import DeepDiff
 
 from ..bbot_fixtures import *
 
@@ -430,19 +431,6 @@ async def test_web_http_compare(blasthttp_mock, bbot_scanner):
     assert compare_helper.compare_body({"a": 1}, {"a": 2}) is False
     assert compare_helper.compare_body({"a": 1}, {"a": 1}) is True
 
-    compare_helper.max_differing_lines = 10
-    shared = [f"line {i}" for i in range(100)]
-    at_threshold = compare_helper.compare_body(
-        shared + [f"a{i}" for i in range(5)],
-        shared + [f"b{i}" for i in range(5)],
-    )
-    over_threshold = compare_helper.compare_body(
-        shared + [f"a{i}" for i in range(6)],
-        shared + [f"b{i}" for i in range(6)],
-    )
-    assert at_threshold is False
-    assert over_threshold is False
-
     compare_helper.max_differing_lines = 500
     base = [f"line {i}" for i in range(20000)]
     similar = [f"line {i} X" if i < 5000 else f"line {i}" for i in range(20000)]
@@ -459,6 +447,97 @@ async def test_web_http_compare(blasthttp_mock, bbot_scanner):
     await scan._cleanup()
 
 
+@pytest.mark.asyncio
+async def test_web_http_compare_filtered_lines_not_counted(blasthttp_mock, bbot_scanner):
+    scan = bbot_scanner()
+    await scan._prep()
+    blasthttp_mock.add_response(url=re.compile(r"http://www\.example\.com.*"), text="wat")
+    compare_helper = scan.helpers.http_compare("http://www.example.com")
+    compare_helper.max_differing_lines = 500
+
+    static = [f"line {i}" for i in range(400)]
+    dynamic_a = [f"nonce {i} A" for i in range(600)]
+    dynamic_b = [f"nonce {i} B" for i in range(600)]
+    dynamic_c = [f"nonce {i} C" for i in range(600)]
+
+    baseline_1 = dynamic_a + static
+    baseline_2 = dynamic_b + static
+    subject = dynamic_c + static
+
+    ddiff = DeepDiff(baseline_1, baseline_2, ignore_order=True, view="tree", threshold_to_diff_deeper=0)
+    compare_helper.ddiff_filters = [x.path() for k in ddiff.keys() for x in list(ddiff[k])]
+    assert len(compare_helper.ddiff_filters) == 600
+
+    assert compare_helper.compare_body(baseline_1, subject) is True
+
+    await scan._cleanup()
+
+
+@pytest.mark.asyncio
+async def test_web_http_compare_bounds_dict_bodies(blasthttp_mock, bbot_scanner):
+    scan = bbot_scanner()
+    await scan._prep()
+    blasthttp_mock.add_response(url=re.compile(r"http://www\.example\.com.*"), text="wat")
+    compare_helper = scan.helpers.http_compare("http://www.example.com")
+    compare_helper.max_differing_lines = 500
+    compare_helper.ddiff_filters = []
+
+    def rows(salt):
+        return [
+            f'<div class="r{i}" tok="{salt}{i}">item {i}{salt}</div>'
+            if i < 1000
+            else f'<div class="r{i}">item {i}</div>'
+            for i in range(4000)
+        ]
+
+    content_1 = {"html": {"body": {"div": rows("a")}}}
+    content_2 = {"html": {"body": {"div": rows("b")}}}
+
+    start = time.monotonic()
+    assert compare_helper.compare_body(content_1, content_2) is False
+    assert (time.monotonic() - start) < 5
+
+    await scan._cleanup()
+
+
+@pytest.mark.asyncio
+async def test_web_http_compare_threshold_boundary(blasthttp_mock, bbot_scanner):
+    scan = bbot_scanner()
+    await scan._prep()
+    blasthttp_mock.add_response(url=re.compile(r"http://www\.example\.com.*"), text="wat")
+    compare_helper = scan.helpers.http_compare("http://www.example.com")
+    compare_helper.max_differing_lines = 10
+
+    static = [f"line {i}" for i in range(100)]
+    at_threshold_1 = [f"nonce {i} A" for i in range(5)] + static
+    at_threshold_2 = [f"nonce {i} B" for i in range(5)] + static
+
+    ddiff = DeepDiff(at_threshold_1, at_threshold_2, ignore_order=True, view="tree", threshold_to_diff_deeper=0)
+    compare_helper.ddiff_filters = [x.path() for k in ddiff.keys() for x in list(ddiff[k])]
+
+    assert compare_helper.compare_body(at_threshold_1, at_threshold_2) is True
+
+    over_threshold_1 = [f"nonce {i} A" for i in range(6)] + static
+    over_threshold_2 = [f"nonce {i} B" for i in range(6)] + static
+    compare_helper.ddiff_filters = []
+    assert compare_helper.compare_body(over_threshold_1, over_threshold_2) is False
+
+    await scan._cleanup()
+
+
+@pytest.mark.asyncio
+async def test_web_http_compare_null_threshold_config(blasthttp_mock, bbot_scanner):
+    scan = bbot_scanner(config={"web": {"http_compare_max_differing_lines": None}})
+    await scan._prep()
+    blasthttp_mock.add_response(url=re.compile(r"http://www\.example\.com.*"), text="wat")
+    compare_helper = scan.helpers.http_compare("http://www.example.com")
+    compare_helper.ddiff_filters = []
+
+    assert compare_helper.compare_body(["a", "b", "c"], ["a", "b", "x"]) is False
+
+    await scan._cleanup()
+
+
 @pytest.mark.asyncio
 async def test_http_proxy(bbot_scanner, bbot_httpserver, proxy_server):
     endpoint = "/test_http_proxy"

From 45cdb99e437bcffbd6bf3331d4042622155aed58 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 09:12:53 -0700
Subject: [PATCH 05/78] fix(diff): count leaves outside ddiff_filters, and
 bound dict bodies

The guard counted raw list items, which broke two ways.

It ignored ddiff_filters, the paths the baseline established as
dynamic. Those lines are excluded from the DeepDiff that follows, but
they were still counted toward the threshold, so a page with enough
per-request nonces reported its body as changed on every probe. That
generates findings rather than suppressing them.

It also required both sides to be lists, but xmltodict.parse succeeds
on ordinary HTML, so those bodies arrive as dicts and dropped into the
unbounded quadratic DeepDiff the guard exists to prevent.

_leaf_counts walks either shape, skips any subtree whose path is
already filtered, and counts leaves, so the threshold sees the same
content the diff will.
---
 bbot/core/helpers/diff.py | 30 +++++++++++++++++++++++-------
 1 file changed, 23 insertions(+), 7 deletions(-)

diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py
index 08a5ef04e2..3e71412303 100644
--- a/bbot/core/helpers/diff.py
+++ b/bbot/core/helpers/diff.py
@@ -237,17 +237,33 @@ def compare_headers(self, headers_1, headers_2):
                 differing_headers.append(header_value)
         return differing_headers
 
+    def _leaf_counts(self, content):
+        counts = Counter()
+        stack = [("root", content)]
+        while stack:
+            path, node = stack.pop()
+            if path in self.ddiff_filters:
+                continue
+            if isinstance(node, dict):
+                for key, value in node.items():
+                    stack.append((f"{path}[{key!r}]", value))
+            elif isinstance(node, list):
+                for index, value in enumerate(node):
+                    stack.append((f"{path}[{index}]", value))
+            else:
+                counts[str(node)] += 1
+        return counts
+
     def compare_body(self, content_1, content_2):
         if content_1 == content_2:
             return True
 
-        if isinstance(content_1, list) and isinstance(content_2, list):
-            counts_1 = Counter(content_1)
-            counts_2 = Counter(content_2)
-            differing = counts_1 - counts_2
-            differing.update(counts_2 - counts_1)
-            if sum(differing.values()) > self.max_differing_lines:
-                return False
+        counts_1 = self._leaf_counts(content_1)
+        counts_2 = self._leaf_counts(content_2)
+        differing = counts_1 - counts_2
+        differing.update(counts_2 - counts_1)
+        if sum(differing.values()) > self.max_differing_lines:
+            return False
 
         ddiff = DeepDiff(
             content_1,

From b50f65fa5f6b5bfd036d5a100a33e00c7537200e Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 09:13:32 -0700
Subject: [PATCH 06/78] fix(diff): coerce a null differing-lines threshold to
 the default

http_compare_max_differing_lines is Optional[int], so an explicit null
is valid config. dict.get returns None for present-but-null, not the
default, and the threshold comparison then raised TypeError on every
body compare.
---
 bbot/core/helpers/diff.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py
index 3e71412303..8911edd320 100644
--- a/bbot/core/helpers/diff.py
+++ b/bbot/core/helpers/diff.py
@@ -100,7 +100,7 @@ def __init__(
         self.headers = headers
         self.cookies = cookies
         self.timeout = 10
-        self.max_differing_lines = self.parent_helper.web_config.get("http_compare_max_differing_lines", 500)
+        self.max_differing_lines = self.parent_helper.web_config.get("http_compare_max_differing_lines", 500) or 500
         # Optional async callback fired once with baseline_1 after the baseline is established.
         self.on_baseline_ready = on_baseline_ready
 

From 4b4e8f778e15b383df40d25577df046b1a8625ea Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 16:00:05 -0700
Subject: [PATCH 07/78] perf(ci): cut test suite wall time via teardown, poll,
 and dep-install fixes

The suite spent most of its wall time waiting rather than working. Each
fix targets a specific source of idle time:

test harness:
- FastShutdownHTTPServer overrides thread_target to poll every 5ms instead
  of socketserver's 0.5s default, so every httpserver fixture teardown no
  longer pays half a second
- stop_server busy-wait drops from 100ms to 5ms per check
- remove an unconditional 0.5s sleep in test_manager_scope_accuracy that
  waited on module init the scan already guarantees

scanner:
- main scan loop polls at 2ms while events are flowing and backs off
  exponentially to 100ms when idle, instead of a flat 100ms sleep that
  throttled every event batch

helpers:
- cache the CloudCheck instance and the wordninja LanguageModel, both of
  which were rebuilt per scan despite identical inputs
- search_format_dict short-circuits when no placeholder is present and
  uses a single compiled regex pass instead of one str.replace per kwarg

deps installer:
- batch every module's pip deps into one resolver pass before the per
  module loop, so pip resolves once instead of once per module. Modules
  with custom pip_constraints are excluded since they must resolve
  against their own set.
---
 bbot/core/helpers/depsinstaller/installer.py  | 37 +++++++++++++++++++
 bbot/core/helpers/helper.py                   | 13 +++++--
 bbot/core/helpers/misc.py                     |  9 +++--
 bbot/core/helpers/wordcloud.py                |  8 +++-
 bbot/scanner/scanner.py                       | 10 ++++-
 bbot/test/conftest.py                         | 19 ++++++++--
 .../test_manager_scope_accuracy.py            |  2 -
 7 files changed, 83 insertions(+), 15 deletions(-)

diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index 82579b4396..4ee373b4f2 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -156,6 +156,7 @@ async def install(self, *modules):
         await self.install_core_deps()
         succeeded = []
         failed = []
+        await self._batch_pip_install(modules)
         try:
             notified = False
             for m in modules:
@@ -472,6 +473,42 @@ def _core_dep_satisfied(self, command):
             )
         return bool(self.parent_helper.which(command))
 
+    async def _batch_pip_install(self, modules):
+        """Pre-install every module's pip deps in one resolver pass.
+
+        install_module() still runs per module afterward; by then the packages are
+        already present, so its own pip call is a no-op. Only modules using the
+        default constraints are batched, since custom constraints must be resolved
+        against their own set.
+        """
+        if self.deps_behavior == "disable":
+            return
+
+        packages = []
+        seen = set()
+        for m in modules:
+            preloaded = self.all_modules_preloaded.get(m)
+            if not preloaded:
+                continue
+            deps = preloaded.get("deps", {})
+            if deps.get("pip_constraints"):
+                continue
+            for dep in deps.get("pip", []):
+                if dep in seen:
+                    continue
+                seen.add(dep)
+                if self.deps_behavior != "force_install":
+                    satisfied, _ = self._pip_deps_satisfied([dep])
+                    if satisfied:
+                        continue
+                packages.append(dep)
+
+        if len(packages) < 2:
+            return
+
+        log.verbose(f"Batch-installing {len(packages):,} pip packages for {len(modules):,} modules")
+        await self.pip_install(packages)
+
     def _pip_deps_satisfied(self, deps_pip):
         """Check whether a module's pip dependencies are currently installed in this environment.
 
diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py
index a4bf2a1572..e3fdaa1409 100644
--- a/bbot/core/helpers/helper.py
+++ b/bbot/core/helpers/helper.py
@@ -7,7 +7,7 @@
 import logging
 from pathlib import Path
 import multiprocessing as mp
-from functools import partial
+from functools import partial, lru_cache
 from concurrent.futures import ProcessPoolExecutor, ThreadPoolExecutor
 
 from . import misc
@@ -30,6 +30,13 @@
 _PR_SET_PDEATHSIG = 1
 
 
+@lru_cache(maxsize=2)
+def _shared_cloudcheck(ssl_verify):
+    from cloudcheck import CloudCheck
+
+    return CloudCheck(verify_ssl=ssl_verify)
+
+
 def _pool_worker_init():
     """Set PR_SET_PDEATHSIG so pool workers die when the parent process dies.
 
@@ -153,10 +160,8 @@ def blasthttp(self):
     @property
     def cloudcheck(self):
         if self._cloudcheck is None:
-            from cloudcheck import CloudCheck
-
             ssl_verify = self.web_config.get("ssl_verify_infrastructure", True)
-            self._cloudcheck = CloudCheck(verify_ssl=ssl_verify)
+            self._cloudcheck = _shared_cloudcheck(ssl_verify)
         return self._cloudcheck
 
     def bloom_filter(self, size):
diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index 1baf3da426..969df0f287 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -1441,6 +1441,9 @@ def search_dict_by_key(key, d):
             yield from search_dict_by_key(key, v)
 
 
+_PLACEHOLDER_REGEX = re.compile(r"#\{(\w+)\}")
+
+
 def search_format_dict(d, **kwargs):
     """Recursively format string values in a dictionary or list using the provided keyword arguments.
 
@@ -1460,9 +1463,9 @@ def search_format_dict(d, **kwargs):
     elif isinstance(d, list):
         return [search_format_dict(v, **kwargs) for v in d]
     elif isinstance(d, str):
-        for find, replace in kwargs.items():
-            find = "#{" + str(find) + "}"
-            d = d.replace(find, replace)
+        if "#{" not in d:
+            return d
+        return _PLACEHOLDER_REGEX.sub(lambda m: kwargs.get(m.group(1), m.group(0)), d)
     return d
 
 
diff --git a/bbot/core/helpers/wordcloud.py b/bbot/core/helpers/wordcloud.py
index a5d9b9aaaf..9f3da3f949 100644
--- a/bbot/core/helpers/wordcloud.py
+++ b/bbot/core/helpers/wordcloud.py
@@ -6,12 +6,18 @@
 from pathlib import Path
 from contextlib import suppress
 from collections import OrderedDict
+from functools import lru_cache
 
 from .misc import tldextract, extract_words
 
 log = logging.getLogger("bbot.core.helpers.wordcloud")
 
 
+@lru_cache(maxsize=8)
+def _load_wordninja_model(wordlist_path):
+    return wordninja.LanguageModel(wordlist_path)
+
+
 class WordCloud(dict):
     """
     WordCloud is a specialized dictionary-like class for storing and aggregating
@@ -485,7 +491,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
         wordlist_dir = Path(__file__).parent.parent.parent / "wordlists"
         wordninja_dns_wordlist = wordlist_dir / "wordninja_dns.txt.gz"
-        self.model = wordninja.LanguageModel(wordninja_dns_wordlist)
+        self.model = _load_wordninja_model(wordninja_dns_wordlist)
 
     def mutations(self, words, max_mutations=None):
         if isinstance(words, str):
diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index 2379dcd940..fc1f7c4765 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -238,6 +238,10 @@ def __init__(
             self.dispatcher = dispatcher
         self.dispatcher.set_scan(self)
 
+        # main loop polls fast when events are flowing, backs off when idle
+        self._main_loop_poll_min = 0.002
+        self._main_loop_poll_max = 0.1
+
         # scope distance
         self.scope_config = self.config.get("scope", {})
         self.scope_search_distance = max(0, int(self.scope_config.get("search_distance", 0)))
@@ -472,6 +476,7 @@ async def async_start(self):
             )
 
             # main scan loop
+            poll_interval = self._main_loop_poll_min
             while 1:
                 # abort if we're aborting
                 if self.aborting:
@@ -484,6 +489,7 @@ async def async_start(self):
                     for e in events:
                         yield e
                     if events:
+                        poll_interval = self._main_loop_poll_min
                         continue
 
                 # break if initialization finished and the scan is no longer active
@@ -492,8 +498,10 @@ async def async_start(self):
                     if not new_activity:
                         self._success = True
                         break
+                    poll_interval = self._main_loop_poll_min
 
-                await asyncio.sleep(0.1)
+                await asyncio.sleep(poll_interval)
+                poll_interval = min(poll_interval * 2, self._main_loop_poll_max)
 
             self._success = True
 
diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index d7caf4aac5..de01d3ca52 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -10,6 +10,17 @@
 from contextlib import suppress
 from pytest_httpserver import HTTPServer
 
+
+class FastShutdownHTTPServer(HTTPServer):
+    """socketserver polls every 0.5s for a shutdown request, so every teardown paid
+    half a second. The poll interval is the only thing standing between us and an
+    immediate stop, and thread_target is documented as the override point."""
+
+    SHUTDOWN_POLL_INTERVAL = 0.005
+
+    def thread_target(self) -> None:
+        self.server.serve_forever(poll_interval=self.SHUTDOWN_POLL_INTERVAL)
+
 from bbot.test.worker import (
     BBOT_TEST_DIR,
     HTTPSERVER_ALLINTERFACES_PORT,
@@ -114,12 +125,12 @@ def patched(self, module_name):
 def stop_server(server):
     server.stop()
     while server.is_running():
-        time.sleep(0.1)  # Wait a bit before checking again
+        time.sleep(0.005)
 
 
 @pytest.fixture
 def bbot_httpserver():
-    server = HTTPServer(host="127.0.0.1", port=HTTPSERVER_PORT, threaded=True)
+    server = FastShutdownHTTPServer(host="127.0.0.1", port=HTTPSERVER_PORT, threaded=True)
     server.start()
 
     yield server
@@ -138,7 +149,7 @@ def bbot_httpserver_ssl():
     keyfile = str(current_dir / "testsslkey.pem")
     certfile = str(current_dir / "testsslcert.pem")
     context.load_cert_chain(certfile, keyfile)
-    server = HTTPServer(host="127.0.0.1", port=HTTPSERVER_SSL_PORT, ssl_context=context, threaded=True)
+    server = FastShutdownHTTPServer(host="127.0.0.1", port=HTTPSERVER_SSL_PORT, ssl_context=context, threaded=True)
     server.start()
 
     yield server
@@ -245,7 +256,7 @@ async def patched_request_batch_stream(self, urls, threads=10, **kwargs):
 
 @pytest.fixture
 def bbot_httpserver_allinterfaces():
-    server = HTTPServer(host="0.0.0.0", port=HTTPSERVER_ALLINTERFACES_PORT, threaded=True)
+    server = FastShutdownHTTPServer(host="0.0.0.0", port=HTTPSERVER_ALLINTERFACES_PORT, threaded=True)
     server.start()
 
     yield server
diff --git a/bbot/test/test_step_1/test_manager_scope_accuracy.py b/bbot/test/test_step_1/test_manager_scope_accuracy.py
index 66d80773bf..7a4b61e472 100644
--- a/bbot/test/test_step_1/test_manager_scope_accuracy.py
+++ b/bbot/test/test_step_1/test_manager_scope_accuracy.py
@@ -129,8 +129,6 @@ async def do_scan(*args, _config={}, _dns_mock={}, scan_callback=None, **kwargs)
         if scan_callback is not None:
             scan_callback(scan)
         output_events = [e async for e in scan.async_start()]
-        # let modules initialize
-        await asyncio.sleep(0.5)
         return (output_events, dummy_module.events, dummy_module_nodupes.events, dummy_graph_output_module.events, dummy_graph_batch_output_module.events)
 
     dns_mock_chain = {

From 1fff8de0428a0a14cb35c0d00b4c021be4fe5333 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 16:04:44 -0700
Subject: [PATCH 08/78] style(test): move FastShutdownHTTPServer below the
 import block

ruff format rejected the class sitting between two import groups. It
belongs after the imports regardless, so move it there rather than
padding the original spot with a blank line.
---
 bbot/test/conftest.py | 22 ++++++++++++----------
 1 file changed, 12 insertions(+), 10 deletions(-)

diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index de01d3ca52..b417b4a551 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -11,16 +11,6 @@
 from pytest_httpserver import HTTPServer
 
 
-class FastShutdownHTTPServer(HTTPServer):
-    """socketserver polls every 0.5s for a shutdown request, so every teardown paid
-    half a second. The poll interval is the only thing standing between us and an
-    immediate stop, and thread_target is documented as the override point."""
-
-    SHUTDOWN_POLL_INTERVAL = 0.005
-
-    def thread_target(self) -> None:
-        self.server.serve_forever(poll_interval=self.SHUTDOWN_POLL_INTERVAL)
-
 from bbot.test.worker import (
     BBOT_TEST_DIR,
     HTTPSERVER_ALLINTERFACES_PORT,
@@ -32,6 +22,18 @@ def thread_target(self) -> None:
 from bbot.core.helpers.misc import execute_sync_or_async
 from bbot.core.helpers.interactsh import server_list as interactsh_servers
 
+
+class FastShutdownHTTPServer(HTTPServer):
+    """socketserver polls every 0.5s for a shutdown request, so every teardown paid
+    half a second. The poll interval is the only thing standing between us and an
+    immediate stop, and thread_target is documented as the override point."""
+
+    SHUTDOWN_POLL_INTERVAL = 0.005
+
+    def thread_target(self) -> None:
+        self.server.serve_forever(poll_interval=self.SHUTDOWN_POLL_INTERVAL)
+
+
 # silence stdout + trace
 root_logger = logging.getLogger()
 pytest_debug_file = Path(__file__).parent.parent.parent / "pytest_debug.log"

From 919993c3c8129e7c0befc47ee3147c33eda33200 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 16:48:00 -0700
Subject: [PATCH 09/78] perf(ci): run distro tests in parallel

distro_tests.yml invoked pytest with no -n, so all six distro containers
ran the entire suite serially at ~32 minutes each, while tests.yml ran
the same suite in parallel at ~15. The distro matrix, not the test
matrix, was setting the wall time for the whole PR.

Use the same -n/--dist loadgroup flags tests.yml already uses.
worker_count.py bounds the count by cores and available RAM, so the
container gets a worker count it can actually feed.
---
 .github/workflows/distro_tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml
index 66ca3f03a4..1e357c289c 100644
--- a/.github/workflows/distro_tests.yml
+++ b/.github/workflows/distro_tests.yml
@@ -51,7 +51,7 @@ jobs:
           uv python install 3.12
           uv python pin 3.12
           uv sync --group dev
-          uv run pytest --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
+          uv run pytest -n $(uv run python bbot/test/worker_count.py) --dist loadgroup --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v7

From 96a63af0eae329835cd1c140e5ef85781b900824 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 16:54:51 -0700
Subject: [PATCH 10/78] perf(ci): scope distro tests to install and runtime
 surface

The distro matrix ran the full 151-module suite on all six containers.
Module tests exercise module logic, which does not vary by distro and is
already covered five times over by the Python matrix in tests.yml. Six
redundant full-suite runs were setting the wall time for every PR.

The question the distro matrix exists to answer is narrower: does bbot
install, resolve deps, and run here. Scope it to the files that actually
probe that surface (e2e, cli, depsinstaller, command, files, python_api,
config, dns) and let tests.yml own module coverage.
---
 .github/workflows/distro_tests.yml | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml
index 1e357c289c..7e56d304cc 100644
--- a/.github/workflows/distro_tests.yml
+++ b/.github/workflows/distro_tests.yml
@@ -51,7 +51,15 @@ jobs:
           uv python install 3.12
           uv python pin 3.12
           uv sync --group dev
-          uv run pytest -n $(uv run python bbot/test/worker_count.py) --dist loadgroup --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
+          uv run pytest -n $(uv run python bbot/test/worker_count.py) --dist loadgroup --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO \
+            bbot/test/test_step_1/test_e2e.py \
+            bbot/test/test_step_1/test_cli.py \
+            bbot/test/test_step_1/test_depsinstaller.py \
+            bbot/test/test_step_1/test_command.py \
+            bbot/test/test_step_1/test_files.py \
+            bbot/test/test_step_1/test_python_api.py \
+            bbot/test/test_step_1/test_config.py \
+            bbot/test/test_step_1/test_dns.py
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v7

From 3f31acef3b72c4de39efeca192f3656c76844e2c Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 17:14:55 -0700
Subject: [PATCH 11/78] perf(test): split test_cli_args into four independent
 tests

test_cli_args was a single 292s test wrapping roughly forty independent
cli._main() invocations. xdist cannot split one test, so it set a hard
floor on wall time no matter how many workers were available.

Each section already set argv, called, and asserted in isolation, so the
split is mechanical. --install-all-deps gets its own test since it
dominates the runtime and now occupies a worker without blocking the
other three sections.
---
 bbot/test/test_step_1/test_cli.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index 2db0037d12..82759b1d44 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -206,6 +206,14 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     assert "[INFO]" in open(scan_log).read()
     shutil.rmtree(output_dir)
 
+
+@pytest.mark.asyncio
+async def test_cli_args_module_options(monkeypatch, caplog, capsys, clean_default_config):
+    caplog.set_level(logging.INFO)
+
+    monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
+    monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
+
     # list module options
     monkeypatch.setattr("sys.argv", ["bbot", "--list-module-options"])
     result = await cli._main()
@@ -323,6 +331,14 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     assert "| dnsbrute " not in out
     assert "| http " in out
 
+
+@pytest.mark.asyncio
+async def test_cli_args_output_modules(monkeypatch, caplog, capsys, clean_default_config):
+    caplog.set_level(logging.INFO)
+
+    monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
+    monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
+
     # -om is additive (defaults stay)
     caplog.clear()
     assert not caplog.text
@@ -418,6 +434,14 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     result = await cli._main()
     assert result is True, "-m dotnetnuke should run without any special flags"
 
+
+@pytest.mark.asyncio
+async def test_cli_args_install_all_deps(monkeypatch, caplog, capsys, clean_default_config):
+    caplog.set_level(logging.INFO)
+
+    monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
+    monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
+
     # install all deps
     monkeypatch.setattr("sys.argv", ["bbot", "--install-all-deps"])
     success = await cli._main()

From b2ae7b6262692a1c39a150ccf8f4bd34f737e7a5 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 17:33:24 -0700
Subject: [PATCH 12/78] perf(test): shard test_module_loading and perdomainonly
 across workers

Both tests walked all 116 scan modules in a single body, so xdist could
not split them and they sat at 259s and 246s respectively, setting a
floor on wall time alongside test_cli_args.

Parametrize both over a module shard so each case loads its own subset.
The union of the shards is the full module list, verified for both the
scan and output sets, so no module loses coverage.

Four shards measured better than eight: each shard pays scanner init
once, so past a point the fixed cost outweighs the split.
---
 bbot/test/test_step_1/test_modules_basic.py | 25 ++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 60e4080714..a300bf7bfd 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -7,6 +7,19 @@
 from bbot.modules.report.base import BaseReportModule
 from bbot.modules.internal.base import BaseInternalModule
 
+MODULE_SHARDS = 4
+
+
+def module_shard(modules, shard):
+    """Slice a module list so each shard loads its own subset.
+
+    These tests walk every module in a single body, which xdist cannot split.
+    Sharding the module list turns one long test into several short ones that
+    together still cover every module.
+    """
+    ordered = sorted(modules)
+    return [m for i, m in enumerate(ordered) if i % MODULE_SHARDS == shard]
+
 
 @pytest.mark.asyncio
 async def test_modules_basic_checks(events, blasthttp_mock):
@@ -337,10 +350,11 @@ class mod_domain_only(BaseModule):
 
 
 @pytest.mark.asyncio
-async def test_modules_basic_perdomainonly(bbot_scanner, monkeypatch):
+@pytest.mark.parametrize("shard", range(MODULE_SHARDS))
+async def test_modules_basic_perdomainonly(bbot_scanner, monkeypatch, shard):
     per_domain_scan = bbot_scanner(
         "evilcorp.com",
-        modules=list(available_modules),
+        modules=module_shard(available_modules, shard),
         config={i: True for i in available_internal_modules if i != "dnsresolve"},
         force_start=True,
     )
@@ -517,10 +531,11 @@ async def handle_event(self, event):
 
 
 @pytest.mark.asyncio
-async def test_module_loading(bbot_scanner):
+@pytest.mark.parametrize("shard", range(MODULE_SHARDS))
+async def test_module_loading(bbot_scanner, shard):
     scan2 = bbot_scanner(
-        modules=list(available_modules),
-        output_modules=list(available_output_modules),
+        modules=module_shard(available_modules, shard),
+        output_modules=module_shard(available_output_modules, shard),
         config={i: True for i in available_internal_modules if i != "dnsresolve"},
         force_start=True,
     )

From eb3c58d7bf75870f004ce7f780f019e9326ff435 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 17:57:21 -0700
Subject: [PATCH 13/78] perf(ci): oversubscribe cores when sizing xdist workers

worker_count capped at core count, so GitHub's 4-core runners ran 4
workers while every worker sat waiting on subprocesses, local HTTP
servers, and DNS. The suite is I/O-bound, so cores are the wrong ceiling.

Double the cpu ceiling and leave the memory bound untouched, since memory
is what actually OOM-kills a run. A 4-core runner now gets 8 workers; the
16-core/16GB case the memory guard was written for still resolves to 16.
---
 bbot/test/worker_count.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/bbot/test/worker_count.py b/bbot/test/worker_count.py
index 754a2b2432..21272463e7 100644
--- a/bbot/test/worker_count.py
+++ b/bbot/test/worker_count.py
@@ -6,6 +6,10 @@
 wants ~2GB on top, so a 16-core/16GB machine running 16 workers gets its
 container OOM-killed (exit 137) and takes the run down with it.
 
+The suite is I/O-bound, not CPU-bound: workers spend most of their time waiting
+on subprocesses, local HTTP servers, and DNS. Capping at core count leaves the
+box idle, so oversubscribe cores and let memory be the real ceiling.
+
 Override with BBOT_TEST_WORKERS=<n>.
 """
 
@@ -14,6 +18,8 @@
 MB_PER_WORKER = 700
 # Docker-backed services (elasticsearch ~2GB), the daemon, and the pytest parent.
 RESERVE_MB = 5120
+# Workers idle on I/O, so run more of them than there are cores.
+OVERSUBSCRIBE = 2
 
 
 def cpu_count():
@@ -28,11 +34,12 @@ def worker_count():
     pinned = os.environ.get("BBOT_TEST_WORKERS", "").strip()
     if pinned:
         return max(1, int(pinned))
+    cpu_ceiling = cpu_count() * OVERSUBSCRIBE
     try:
         total_mb = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") // (1024 * 1024)
     except (ValueError, OSError, AttributeError):
-        return cpu_count()
-    return max(1, min(cpu_count(), (total_mb - RESERVE_MB) // MB_PER_WORKER))
+        return cpu_ceiling
+    return max(1, min(cpu_ceiling, (total_mb - RESERVE_MB) // MB_PER_WORKER))
 
 
 if __name__ == "__main__":

From df3cdb8cdfb2193a6ec1f1cb8bea5fa9edfb7c41 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 18:14:45 -0700
Subject: [PATCH 14/78] test(ci): update worker_count guards for core
 oversubscription

The core-cap guards pinned the old ceiling and failed once workers were
oversubscribed. Assert against OVERSUBSCRIBE rather than a literal so the
guards track the constant instead of restating it.

The memory-cap and never-zero guards are untouched: memory is still the
bound that prevents an OOM-killed run.
---
 bbot/test/test_step_1/test_worker_count.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bbot/test/test_step_1/test_worker_count.py b/bbot/test/test_step_1/test_worker_count.py
index e6c50f231f..5f2c25fe99 100644
--- a/bbot/test/test_step_1/test_worker_count.py
+++ b/bbot/test/test_step_1/test_worker_count.py
@@ -1,6 +1,6 @@
 from unittest.mock import patch
 
-from bbot.test.worker_count import RESERVE_MB, MB_PER_WORKER, cpu_count, worker_count
+from bbot.test.worker_count import OVERSUBSCRIBE, RESERVE_MB, MB_PER_WORKER, cpu_count, worker_count
 
 
 def _sysconf(total_mb):
@@ -44,11 +44,11 @@ def test_worker_count_is_capped_by_memory():
                 assert worker_count() == 2
 
 
-def test_worker_count_is_capped_by_cores():
+def test_worker_count_is_capped_by_oversubscribed_cores():
     with patch.dict("os.environ", {}, clear=True):
         with patch("bbot.test.worker_count.cpu_count", return_value=4):
             with patch("os.sysconf", _sysconf(RESERVE_MB + 64 * MB_PER_WORKER)):
-                assert worker_count() == 4
+                assert worker_count() == 4 * OVERSUBSCRIBE
 
 
 def test_worker_count_never_returns_zero_on_small_machines():
@@ -64,7 +64,7 @@ def test_worker_count_falls_back_to_cpus_when_memory_unknown():
         with patch.dict("os.environ", {}, clear=True):
             with patch("bbot.test.worker_count.cpu_count", return_value=6):
                 with patch("os.sysconf", side_effect=exc):
-                    assert worker_count() == 6
+                    assert worker_count() == 6 * OVERSUBSCRIBE
 
 
 def test_cpu_count_prefers_affinity():

From 21c78e42f6b551f9ff80f41e60b69692340de3de Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 18:15:07 -0700
Subject: [PATCH 15/78] Revert "test(ci): update worker_count guards for core
 oversubscription"

This reverts commit df3cdb8cdfb2193a6ec1f1cb8bea5fa9edfb7c41.
---
 bbot/test/test_step_1/test_worker_count.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/bbot/test/test_step_1/test_worker_count.py b/bbot/test/test_step_1/test_worker_count.py
index 5f2c25fe99..e6c50f231f 100644
--- a/bbot/test/test_step_1/test_worker_count.py
+++ b/bbot/test/test_step_1/test_worker_count.py
@@ -1,6 +1,6 @@
 from unittest.mock import patch
 
-from bbot.test.worker_count import OVERSUBSCRIBE, RESERVE_MB, MB_PER_WORKER, cpu_count, worker_count
+from bbot.test.worker_count import RESERVE_MB, MB_PER_WORKER, cpu_count, worker_count
 
 
 def _sysconf(total_mb):
@@ -44,11 +44,11 @@ def test_worker_count_is_capped_by_memory():
                 assert worker_count() == 2
 
 
-def test_worker_count_is_capped_by_oversubscribed_cores():
+def test_worker_count_is_capped_by_cores():
     with patch.dict("os.environ", {}, clear=True):
         with patch("bbot.test.worker_count.cpu_count", return_value=4):
             with patch("os.sysconf", _sysconf(RESERVE_MB + 64 * MB_PER_WORKER)):
-                assert worker_count() == 4 * OVERSUBSCRIBE
+                assert worker_count() == 4
 
 
 def test_worker_count_never_returns_zero_on_small_machines():
@@ -64,7 +64,7 @@ def test_worker_count_falls_back_to_cpus_when_memory_unknown():
         with patch.dict("os.environ", {}, clear=True):
             with patch("bbot.test.worker_count.cpu_count", return_value=6):
                 with patch("os.sysconf", side_effect=exc):
-                    assert worker_count() == 6 * OVERSUBSCRIBE
+                    assert worker_count() == 6
 
 
 def test_cpu_count_prefers_affinity():

From a23fbac20badb82a709988e6c31df14b5adc9511 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 18:15:07 -0700
Subject: [PATCH 16/78] Revert "perf(ci): oversubscribe cores when sizing xdist
 workers"

This reverts commit eb3c58d7bf75870f004ce7f780f019e9326ff435.
---
 bbot/test/worker_count.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/bbot/test/worker_count.py b/bbot/test/worker_count.py
index 21272463e7..754a2b2432 100644
--- a/bbot/test/worker_count.py
+++ b/bbot/test/worker_count.py
@@ -6,10 +6,6 @@
 wants ~2GB on top, so a 16-core/16GB machine running 16 workers gets its
 container OOM-killed (exit 137) and takes the run down with it.
 
-The suite is I/O-bound, not CPU-bound: workers spend most of their time waiting
-on subprocesses, local HTTP servers, and DNS. Capping at core count leaves the
-box idle, so oversubscribe cores and let memory be the real ceiling.
-
 Override with BBOT_TEST_WORKERS=<n>.
 """
 
@@ -18,8 +14,6 @@
 MB_PER_WORKER = 700
 # Docker-backed services (elasticsearch ~2GB), the daemon, and the pytest parent.
 RESERVE_MB = 5120
-# Workers idle on I/O, so run more of them than there are cores.
-OVERSUBSCRIBE = 2
 
 
 def cpu_count():
@@ -34,12 +28,11 @@ def worker_count():
     pinned = os.environ.get("BBOT_TEST_WORKERS", "").strip()
     if pinned:
         return max(1, int(pinned))
-    cpu_ceiling = cpu_count() * OVERSUBSCRIBE
     try:
         total_mb = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") // (1024 * 1024)
     except (ValueError, OSError, AttributeError):
-        return cpu_ceiling
-    return max(1, min(cpu_ceiling, (total_mb - RESERVE_MB) // MB_PER_WORKER))
+        return cpu_count()
+    return max(1, min(cpu_count(), (total_mb - RESERVE_MB) // MB_PER_WORKER))
 
 
 if __name__ == "__main__":

From 65a3ccc902879ab7dc7691625355db02ab9d92a1 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 18:17:12 -0700
Subject: [PATCH 17/78] perf(ci): shard the test matrix across runners

xdist only parallelises within a single runner, so the suite was pinned to
one 4-core machine per Python version and roughly 3800s of work could not
go faster than about 950s of wall time.

Add BBOT_TEST_SHARDS/BBOT_TEST_SHARD, sharding on the sorted nodeid at
collection time, and fan the matrix out to four shards per version. Every
test lands in exactly one shard, verified against a full collection: 869
tests, union of the four shards, no overlap and nothing dropped.

Deselected tests are reported through pytest_deselected so the counts in
each job stay honest about what ran.
---
 .github/workflows/tests.yml |  5 ++++-
 bbot/test/conftest.py       | 23 +++++++++++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index d81333f76b..79de5aa98f 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -18,6 +18,7 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+        shard: [0, 1, 2, 3]
     steps:
       - uses: actions/checkout@v7
       - name: Set up Python
@@ -37,13 +38,15 @@ jobs:
       - name: Run tests
         env:
           BBOT_IO_API_KEY: ${{ secrets.BBOT_IO_API_KEY }}
+          BBOT_TEST_SHARDS: "4"
+          BBOT_TEST_SHARD: ${{ matrix.shard }}
         run: |
           uv run pytest -vv -n $(python bbot/test/worker_count.py) --dist loadgroup --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: pytest-debug-logs-${{ env.PYTHON_VERSION }}
+          name: pytest-debug-logs-${{ env.PYTHON_VERSION }}-${{ matrix.shard }}
           path: pytest_debug.log
       - name: Upload Code Coverage
         uses: codecov/codecov-action@v7
diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index b417b4a551..d9bc1cf72c 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -370,6 +370,24 @@ def proxy_server():
     server_thread.join()
 
 
+def _shard_items(items):
+    """Keep only this shard's slice of the collected tests.
+
+    xdist parallelises within one runner; this splits across runners so CI can
+    use more than one machine's worth of cores. Sharding on the sorted nodeid
+    keeps the slice stable across jobs, so every test lands in exactly one shard.
+    """
+    try:
+        shards = int(os.environ.get("BBOT_TEST_SHARDS", "") or 1)
+        shard = int(os.environ.get("BBOT_TEST_SHARD", "") or 0)
+    except ValueError:
+        return items
+    if shards <= 1 or not (0 <= shard < shards):
+        return items
+    ordered = sorted(items, key=lambda i: i.nodeid)
+    return [item for i, item in enumerate(ordered) if i % shards == shard]
+
+
 def pytest_collection_modifyitems(config, items):
     """Pin docker-backed tests to one xdist worker.
 
@@ -383,6 +401,11 @@ def pytest_collection_modifyitems(config, items):
         if cls is not None and getattr(cls, "skip_distro_tests", False):
             item.add_marker(pytest.mark.xdist_group("docker"))
 
+    kept = _shard_items(items)
+    if len(kept) != len(items):
+        config.hook.pytest_deselected(items=[i for i in items if i not in set(kept)])
+        items[:] = kept
+
 
 def pytest_terminal_summary(terminalreporter, exitstatus, config):  # pragma: no cover
     RED = "\033[1;31m"

From b7fc1a75d9a06f60467594befe06c83f0fef2848 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 18:32:58 -0700
Subject: [PATCH 18/78] fix(test): install neo4j before the test that imports
 it

setup_before_prep monkeypatches neo4j.AsyncGraphDatabase, which imports
neo4j, but the pip_install lived in setup_after_prep and therefore ran
later. The test only passed because something earlier in the session had
already pulled the package in, an ordering dependency that sharding
exposed as ModuleNotFoundError.

Move the install ahead of the patch so the test stands on its own.
Verified against an environment with neo4j uninstalled.
---
 bbot/test/test_step_2/module_tests/test_module_neo4j.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_neo4j.py b/bbot/test/test_step_2/module_tests/test_module_neo4j.py
index be395206d3..27d00b0379 100644
--- a/bbot/test/test_step_2/module_tests/test_module_neo4j.py
+++ b/bbot/test/test_step_2/module_tests/test_module_neo4j.py
@@ -5,13 +5,14 @@ class TestNeo4j(ModuleTestBase):
     config_overrides = {"modules": {"neo4j": {"uri": "bolt://127.0.0.1:11111"}}}
 
     async def setup_after_prep(self, module_test):
-        # install neo4j
-        deps_pip = module_test.preloaded["neo4j"]["deps"]["pip"]
-        await module_test.scan.helpers.depsinstaller.pip_install(deps_pip)
-
         self.neo4j_used = False
 
     async def setup_before_prep(self, module_test):
+        # monkeypatching neo4j below imports it, so install it first rather than
+        # relying on another test in the session having already pulled it in
+        deps_pip = module_test.preloaded["neo4j"]["deps"]["pip"]
+        await module_test.scan.helpers.depsinstaller.pip_install(deps_pip)
+
         class MockResult:
             async def data(s):
                 self.neo4j_used = True

From 4326dcd2c541a90173de6648ce86774c836ced7f Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 19:01:14 -0700
Subject: [PATCH 19/78] fix(ci): restore required test contexts after shard
 fanout

Sharding added a second matrix dimension, renaming every job from
"test (3.10)" to "test (3.10, 0)". The protecc ruleset on dev
requires the un-sharded context names, so those four contexts were
never reported and PRs hung on Expected forever.

Add a test_gate job matrixed on python-version only, named to emit
the exact required contexts, gated on the sharded matrix result.
---
 .github/workflows/tests.yml | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 79de5aa98f..12e0208153 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -54,6 +54,22 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./cov.xml
           verbose: true
+  test_gate:
+    name: test (${{ matrix.python-version }})
+    needs: test
+    if: always()
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
+    steps:
+      - name: Check shard results
+        run: |
+          if [[ "${{ needs.test.result }}" != "success" ]]; then
+            echo "sharded test matrix result: ${{ needs.test.result }}"
+            exit 1
+          fi
   publish_code:
     needs: test
     runs-on: ubuntu-latest

From 707e3de82b0c391458acf5202725db4cb4fbd643 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 19:06:03 -0700
Subject: [PATCH 20/78] Revert "fix(ci): restore required test contexts after
 shard fanout"

This reverts commit 4326dcd2c541a90173de6648ce86774c836ced7f.
---
 .github/workflows/tests.yml | 16 ----------------
 1 file changed, 16 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 12e0208153..79de5aa98f 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -54,22 +54,6 @@ jobs:
           token: ${{ secrets.CODECOV_TOKEN }}
           files: ./cov.xml
           verbose: true
-  test_gate:
-    name: test (${{ matrix.python-version }})
-    needs: test
-    if: always()
-    runs-on: ubuntu-latest
-    strategy:
-      fail-fast: false
-      matrix:
-        python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
-    steps:
-      - name: Check shard results
-        run: |
-          if [[ "${{ needs.test.result }}" != "success" ]]; then
-            echo "sharded test matrix result: ${{ needs.test.result }}"
-            exit 1
-          fi
   publish_code:
     needs: test
     runs-on: ubuntu-latest

From a26cea4d601432ea69d06ba247903ae0e7efd9bc Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 19:06:03 -0700
Subject: [PATCH 21/78] Revert "perf(ci): shard the test matrix across runners"

This reverts commit 65a3ccc902879ab7dc7691625355db02ab9d92a1.
---
 .github/workflows/tests.yml |  5 +----
 bbot/test/conftest.py       | 23 -----------------------
 2 files changed, 1 insertion(+), 27 deletions(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index 79de5aa98f..d81333f76b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -18,7 +18,6 @@ jobs:
       fail-fast: false
       matrix:
         python-version: ["3.10", "3.11", "3.12", "3.13", "3.14"]
-        shard: [0, 1, 2, 3]
     steps:
       - uses: actions/checkout@v7
       - name: Set up Python
@@ -38,15 +37,13 @@ jobs:
       - name: Run tests
         env:
           BBOT_IO_API_KEY: ${{ secrets.BBOT_IO_API_KEY }}
-          BBOT_TEST_SHARDS: "4"
-          BBOT_TEST_SHARD: ${{ matrix.shard }}
         run: |
           uv run pytest -vv -n $(python bbot/test/worker_count.py) --dist loadgroup --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v7
         with:
-          name: pytest-debug-logs-${{ env.PYTHON_VERSION }}-${{ matrix.shard }}
+          name: pytest-debug-logs-${{ env.PYTHON_VERSION }}
           path: pytest_debug.log
       - name: Upload Code Coverage
         uses: codecov/codecov-action@v7
diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index d9bc1cf72c..b417b4a551 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -370,24 +370,6 @@ def proxy_server():
     server_thread.join()
 
 
-def _shard_items(items):
-    """Keep only this shard's slice of the collected tests.
-
-    xdist parallelises within one runner; this splits across runners so CI can
-    use more than one machine's worth of cores. Sharding on the sorted nodeid
-    keeps the slice stable across jobs, so every test lands in exactly one shard.
-    """
-    try:
-        shards = int(os.environ.get("BBOT_TEST_SHARDS", "") or 1)
-        shard = int(os.environ.get("BBOT_TEST_SHARD", "") or 0)
-    except ValueError:
-        return items
-    if shards <= 1 or not (0 <= shard < shards):
-        return items
-    ordered = sorted(items, key=lambda i: i.nodeid)
-    return [item for i, item in enumerate(ordered) if i % shards == shard]
-
-
 def pytest_collection_modifyitems(config, items):
     """Pin docker-backed tests to one xdist worker.
 
@@ -401,11 +383,6 @@ def pytest_collection_modifyitems(config, items):
         if cls is not None and getattr(cls, "skip_distro_tests", False):
             item.add_marker(pytest.mark.xdist_group("docker"))
 
-    kept = _shard_items(items)
-    if len(kept) != len(items):
-        config.hook.pytest_deselected(items=[i for i in items if i not in set(kept)])
-        items[:] = kept
-
 
 def pytest_terminal_summary(terminalreporter, exitstatus, config):  # pragma: no cover
     RED = "\033[1;31m"

From 216425ad6a8f409634f2dfa939fa7ef73d4c14fd Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 20:34:44 -0700
Subject: [PATCH 22/78] Revert "perf(test): shard test_module_loading and
 perdomainonly across workers"

This reverts commit b2ae7b6262692a1c39a150ccf8f4bd34f737e7a5.
---
 bbot/test/test_step_1/test_modules_basic.py | 25 +++++----------------
 1 file changed, 5 insertions(+), 20 deletions(-)

diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index a300bf7bfd..60e4080714 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -7,19 +7,6 @@
 from bbot.modules.report.base import BaseReportModule
 from bbot.modules.internal.base import BaseInternalModule
 
-MODULE_SHARDS = 4
-
-
-def module_shard(modules, shard):
-    """Slice a module list so each shard loads its own subset.
-
-    These tests walk every module in a single body, which xdist cannot split.
-    Sharding the module list turns one long test into several short ones that
-    together still cover every module.
-    """
-    ordered = sorted(modules)
-    return [m for i, m in enumerate(ordered) if i % MODULE_SHARDS == shard]
-
 
 @pytest.mark.asyncio
 async def test_modules_basic_checks(events, blasthttp_mock):
@@ -350,11 +337,10 @@ class mod_domain_only(BaseModule):
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("shard", range(MODULE_SHARDS))
-async def test_modules_basic_perdomainonly(bbot_scanner, monkeypatch, shard):
+async def test_modules_basic_perdomainonly(bbot_scanner, monkeypatch):
     per_domain_scan = bbot_scanner(
         "evilcorp.com",
-        modules=module_shard(available_modules, shard),
+        modules=list(available_modules),
         config={i: True for i in available_internal_modules if i != "dnsresolve"},
         force_start=True,
     )
@@ -531,11 +517,10 @@ async def handle_event(self, event):
 
 
 @pytest.mark.asyncio
-@pytest.mark.parametrize("shard", range(MODULE_SHARDS))
-async def test_module_loading(bbot_scanner, shard):
+async def test_module_loading(bbot_scanner):
     scan2 = bbot_scanner(
-        modules=module_shard(available_modules, shard),
-        output_modules=module_shard(available_output_modules, shard),
+        modules=list(available_modules),
+        output_modules=list(available_output_modules),
         config={i: True for i in available_internal_modules if i != "dnsresolve"},
         force_start=True,
     )

From af6026564aafde79b1467d45b86d7970271e4d58 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 20:34:44 -0700
Subject: [PATCH 23/78] Revert "perf(test): split test_cli_args into four
 independent tests"

This reverts commit 3f31acef3b72c4de39efeca192f3656c76844e2c.
---
 bbot/test/test_step_1/test_cli.py | 24 ------------------------
 1 file changed, 24 deletions(-)

diff --git a/bbot/test/test_step_1/test_cli.py b/bbot/test/test_step_1/test_cli.py
index 82759b1d44..2db0037d12 100644
--- a/bbot/test/test_step_1/test_cli.py
+++ b/bbot/test/test_step_1/test_cli.py
@@ -206,14 +206,6 @@ async def test_cli_args(monkeypatch, caplog, capsys, clean_default_config):
     assert "[INFO]" in open(scan_log).read()
     shutil.rmtree(output_dir)
 
-
-@pytest.mark.asyncio
-async def test_cli_args_module_options(monkeypatch, caplog, capsys, clean_default_config):
-    caplog.set_level(logging.INFO)
-
-    monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
-    monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
-
     # list module options
     monkeypatch.setattr("sys.argv", ["bbot", "--list-module-options"])
     result = await cli._main()
@@ -331,14 +323,6 @@ async def test_cli_args_module_options(monkeypatch, caplog, capsys, clean_defaul
     assert "| dnsbrute " not in out
     assert "| http " in out
 
-
-@pytest.mark.asyncio
-async def test_cli_args_output_modules(monkeypatch, caplog, capsys, clean_default_config):
-    caplog.set_level(logging.INFO)
-
-    monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
-    monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
-
     # -om is additive (defaults stay)
     caplog.clear()
     assert not caplog.text
@@ -434,14 +418,6 @@ async def test_cli_args_output_modules(monkeypatch, caplog, capsys, clean_defaul
     result = await cli._main()
     assert result is True, "-m dotnetnuke should run without any special flags"
 
-
-@pytest.mark.asyncio
-async def test_cli_args_install_all_deps(monkeypatch, caplog, capsys, clean_default_config):
-    caplog.set_level(logging.INFO)
-
-    monkeypatch.setattr(sys, "exit", lambda *args, **kwargs: True)
-    monkeypatch.setattr(os, "_exit", lambda *args, **kwargs: True)
-
     # install all deps
     monkeypatch.setattr("sys.argv", ["bbot", "--install-all-deps"])
     success = await cli._main()

From 213a5bdd93cfed3ef358bcb61c0551b1e385bdd3 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 20:37:57 -0700
Subject: [PATCH 24/78] perf(test): install module deps once into a shared dir

Every xdist worker got its own BBOT home, and dependency installs are
keyed on that home, so all 126 modules' deps were installed once per
worker instead of once per run. That was the single largest fixed cost
in the suite and it scaled with worker count.

Split the home into two halves:

- cache, tools and lib move to BBOT_SHARED_DEPS_DIR when set. These hold
  install state, downloaded binaries and libs, all of which are keyed on
  content and safe to share between concurrent scans.
- scans, temp and the rest stay per worker, so scan output and the
  sessionfinish cleanup remain isolated exactly as before.

The module hash now keys on the deps directory rather than bbot_home,
otherwise a shared install would still be invalidated per worker.

Concurrent installs into the shared dir are serialized with an flock on
the deps dir, and setup status is re-read after acquiring it, so a worker
that waited picks up what the holder just installed instead of redoing it.
---
 bbot/core/helpers/depsinstaller/installer.py | 26 +++++++++++++++++---
 bbot/core/helpers/helper.py                  | 12 ++++++---
 bbot/test/conftest.py                        |  7 ++++--
 bbot/test/worker.py                          |  6 +++++
 4 files changed, 43 insertions(+), 8 deletions(-)

diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index 4ee373b4f2..1de9da517f 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -2,6 +2,7 @@
 import sys
 import stat
 import json
+import fcntl
 import mmh3
 import orjson
 import shutil
@@ -11,7 +12,7 @@
 from pathlib import Path
 from threading import Lock
 from itertools import chain
-from contextlib import suppress
+from contextlib import contextmanager, suppress
 from secrets import token_bytes
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
@@ -153,6 +154,25 @@ def __init__(self, parent_helper):
         self.ensure_root_lock = Lock()
 
     async def install(self, *modules):
+        # Concurrent scans (notably xdist workers) share the deps dir, so serialize
+        # installs across processes: without this they race on the same files and
+        # each pays the full install anyway.
+        with self._install_lock():
+            # another process may have installed deps while we waited for the lock
+            self.setup_status = self.read_setup_status()
+            return await self._install(*modules)
+
+    @contextmanager
+    def _install_lock(self):
+        lock_file = self.data_dir / "install.lock"
+        with open(lock_file, "w") as f:
+            fcntl.flock(f, fcntl.LOCK_EX)
+            try:
+                yield
+            finally:
+                fcntl.flock(f, fcntl.LOCK_UN)
+
+    async def _install(self, *modules):
         await self.install_core_deps()
         succeeded = []
         failed = []
@@ -172,11 +192,11 @@ async def install(self, *modules):
                 preloaded = self.all_modules_preloaded[m]
                 log.debug(f"Installing {m} - Preloaded Deps {preloaded['deps']}")
                 # make a hash of the dependencies and check if it's already been handled
-                # take into consideration whether the venv or bbot home directory changes
+                # take into consideration whether the venv or the deps directory changes
                 module_hash = self.parent_helper.sha1(
                     json.dumps(preloaded["deps"], sort_keys=True)
                     + self.venv
-                    + str(self.parent_helper.bbot_home)
+                    + str(self.parent_helper.tools_dir.parent)
                     + os.uname()[1]
                     + str(__version__)
                 ).hexdigest()
diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py
index e3fdaa1409..da4bec71c6 100644
--- a/bbot/core/helpers/helper.py
+++ b/bbot/core/helpers/helper.py
@@ -90,10 +90,16 @@ class ConfigAwareHelper:
     def __init__(self, preset):
         self.preset = preset
         self.bbot_home = self.preset.bbot_home
-        self.cache_dir = self.bbot_home / "cache"
+        # Dependency installs, downloaded tools and libs are keyed on these paths and
+        # are safe to share across concurrent scans. BBOT_SHARED_DEPS_DIR lets the test
+        # suite give every xdist worker its own home (for scan output) while still
+        # installing deps exactly once.
+        shared_home = os.environ.get("BBOT_SHARED_DEPS_DIR", "").strip()
+        deps_home = Path(shared_home) if shared_home else self.bbot_home
+        self.cache_dir = deps_home / "cache"
+        self.tools_dir = deps_home / "tools"
+        self.lib_dir = deps_home / "lib"
         self.temp_dir = self.bbot_home / "temp"
-        self.tools_dir = self.bbot_home / "tools"
-        self.lib_dir = self.bbot_home / "lib"
         self.scans_dir = self.bbot_home / "scans"
         self.wordlist_dir = Path(__file__).parent.parent.parent / "wordlists"
         self.current_dir = Path.cwd()
diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index b417b4a551..76b5f81b3b 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -9,10 +9,9 @@
 from pathlib import Path
 from contextlib import suppress
 from pytest_httpserver import HTTPServer
-
-
 from bbot.test.worker import (
     BBOT_TEST_DIR,
+    BBOT_TEST_SHARED_DIR,
     HTTPSERVER_ALLINTERFACES_PORT,
     HTTPSERVER_PORT,
     HTTPSERVER_SSL_PORT,
@@ -51,6 +50,10 @@ def thread_target(self) -> None:
 # and the sessionfinish cleanup below would delete a directory still in use.
 test_config["home"] = str(BBOT_TEST_DIR)
 
+# Deps install once into a shared dir instead of once per worker. The per-worker
+# home above still isolates scan output and temp files.
+os.environ.setdefault("BBOT_SHARED_DEPS_DIR", str(BBOT_TEST_SHARED_DIR))
+
 os.environ["BBOT_DEBUG"] = "True"
 CORE.logger.log_level = logging.DEBUG
 
diff --git a/bbot/test/worker.py b/bbot/test/worker.py
index 8901dc65fb..0167bb1ed8 100644
--- a/bbot/test/worker.py
+++ b/bbot/test/worker.py
@@ -48,6 +48,12 @@ def worker_dir(base=BASE_BBOT_TEST_DIR):
     return Path(f"{base}_{worker}") if worker else Path(base)
 
 
+# Dependency installs are keyed on bbot_home, so a per-worker home makes every
+# worker reinstall all ~126 modules' deps from scratch. Point the install-once
+# dirs at one shared location; scan output and temp files stay per-worker.
+BBOT_TEST_SHARED_DIR = Path(f"{BASE_BBOT_TEST_DIR}_shared")
+
+
 HTTPSERVER_PORT = worker_port(8888)
 HTTPSERVER_SSL_PORT = worker_port(9999)
 HTTPSERVER_ALLINTERFACES_PORT = worker_port(5556)

From e8e4dcefb4ace2ddede7851255a9eeb21ff8a05a Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 20:58:45 -0700
Subject: [PATCH 25/78] fix(test): point core deps dirs at the shared dir too

The shared deps dir was only wired into ConfigAwareHelper, but BBOTCore
exposes its own cache_dir/tools_dir/lib_dir off home, and those are what
set BBOT_TOOLS for the ansible playbooks. Installs therefore targeted a
tools dir under the per-worker home that nothing had created, and every
ansible download task failed with:

    dest '/root/.bbot/tools' must be an existing dir

Move the redirect to BBOTCore.deps_home so core and helper resolve the
same paths, and have the helper and the module hash read it from there
instead of recomputing it. temp and scans stay on home.

Writing the module preload cache is now atomic (temp file plus rename)
since the cache dir is shared between concurrent scans and a reader must
never see a half-written pickle.
---
 bbot/core/core.py                            | 18 +++++++++++++++---
 bbot/core/helpers/depsinstaller/installer.py |  2 +-
 bbot/core/helpers/helper.py                  | 15 ++++++---------
 bbot/core/modules.py                         | 13 +++++++++++--
 bbot/scanner/preset/preset.py                |  4 ++++
 5 files changed, 37 insertions(+), 15 deletions(-)

diff --git a/bbot/core/core.py b/bbot/core/core.py
index 983d489d0f..dd9b9a6357 100644
--- a/bbot/core/core.py
+++ b/bbot/core/core.py
@@ -64,13 +64,25 @@ def _prep_multiprocessing(self):
     def home(self):
         return Path(self.config["home"]).expanduser().resolve()
 
+    @property
+    def deps_home(self):
+        """
+        Install state, downloaded tools and libs are keyed on content, not on the scan,
+        so concurrent scans can share them. BBOT_SHARED_DEPS_DIR points them at one
+        location while everything scan-specific stays under home.
+        """
+        shared = os.environ.get("BBOT_SHARED_DEPS_DIR", "").strip()
+        if shared:
+            return Path(shared).expanduser().resolve()
+        return self.home
+
     @property
     def cache_dir(self):
-        return self.home / "cache"
+        return self.deps_home / "cache"
 
     @property
     def tools_dir(self):
-        return self.home / "tools"
+        return self.deps_home / "tools"
 
     @property
     def temp_dir(self):
@@ -78,7 +90,7 @@ def temp_dir(self):
 
     @property
     def lib_dir(self):
-        return self.home / "lib"
+        return self.deps_home / "lib"
 
     @property
     def scans_dir(self):
diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index 1de9da517f..189d10ed79 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -196,7 +196,7 @@ async def _install(self, *modules):
                 module_hash = self.parent_helper.sha1(
                     json.dumps(preloaded["deps"], sort_keys=True)
                     + self.venv
-                    + str(self.parent_helper.tools_dir.parent)
+                    + str(self.parent_helper.deps_home)
                     + os.uname()[1]
                     + str(__version__)
                 ).hexdigest()
diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py
index da4bec71c6..561cb2a933 100644
--- a/bbot/core/helpers/helper.py
+++ b/bbot/core/helpers/helper.py
@@ -90,15 +90,12 @@ class ConfigAwareHelper:
     def __init__(self, preset):
         self.preset = preset
         self.bbot_home = self.preset.bbot_home
-        # Dependency installs, downloaded tools and libs are keyed on these paths and
-        # are safe to share across concurrent scans. BBOT_SHARED_DEPS_DIR lets the test
-        # suite give every xdist worker its own home (for scan output) while still
-        # installing deps exactly once.
-        shared_home = os.environ.get("BBOT_SHARED_DEPS_DIR", "").strip()
-        deps_home = Path(shared_home) if shared_home else self.bbot_home
-        self.cache_dir = deps_home / "cache"
-        self.tools_dir = deps_home / "tools"
-        self.lib_dir = deps_home / "lib"
+        # deps_home is bbot_home unless BBOT_SHARED_DEPS_DIR redirects the install-once
+        # dirs elsewhere; see BBOTCore.deps_home.
+        self.deps_home = self.preset.deps_home
+        self.cache_dir = self.deps_home / "cache"
+        self.tools_dir = self.deps_home / "tools"
+        self.lib_dir = self.deps_home / "lib"
         self.temp_dir = self.bbot_home / "temp"
         self.scans_dir = self.bbot_home / "scans"
         self.wordlist_dir = Path(__file__).parent.parent.parent / "wordlists"
diff --git a/bbot/core/modules.py b/bbot/core/modules.py
index cee81ef6f2..609a822968 100644
--- a/bbot/core/modules.py
+++ b/bbot/core/modules.py
@@ -467,8 +467,17 @@ def preload_cache(self):
     def preload_cache(self, value):
         self._preload_cache = value
         mkdir(self.preload_cache_file.parent)
-        with open(self.preload_cache_file, "wb") as f:
-            pickle.dump(self._preload_cache, f)
+        # concurrent scans can share the cache dir, so write via a private temp file
+        # and rename: a reader either sees the old cache or the new one, never a
+        # half-written pickle
+        tmp_file = self.preload_cache_file.with_suffix(f".{os.getpid()}.tmp")
+        try:
+            with open(tmp_file, "wb") as f:
+                pickle.dump(self._preload_cache, f)
+            os.replace(tmp_file, self.preload_cache_file)
+        finally:
+            with suppress(OSError):
+                tmp_file.unlink()
 
     def save_preload_cache(self):
         self.preload_cache = self.__preloaded
diff --git a/bbot/scanner/preset/preset.py b/bbot/scanner/preset/preset.py
index ed9e502c6f..6414962565 100644
--- a/bbot/scanner/preset/preset.py
+++ b/bbot/scanner/preset/preset.py
@@ -296,6 +296,10 @@ def __init__(
     def bbot_home(self):
         return Path(self.config.get("home", "~/.bbot")).expanduser().resolve()
 
+    @property
+    def deps_home(self):
+        return self.core.deps_home
+
     @property
     def target(self):
         if self._target is None:

From 69a7aa85aa166cf17c65d5507a4580ae2cb5c805 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 21:22:18 -0700
Subject: [PATCH 26/78] test: assert BBOT_TOOLS against the resolved tools dir

Both tests hardcoded the tools dir as home/tools, which only held while
deps lived under the scan home. With the install-once dirs redirected,
BBOT_TOOLS is the shared path and both failed:

    assert '/tmp/.bbot_test_shared/tools' == '/tmp/.bbot_python_api_test_gw1/tools'
    assert '/tmp/.bbot_test_gw3/tools' in [...]

Assert against helpers.tools_dir so the check follows wherever tools
resolve. test_python_api keeps its home coverage by asserting scans_dir
still lands under the configured home, which is the half of the split
these tests were really pinning.
---
 bbot/test/test_step_1/test_command.py    | 10 +++++-----
 bbot/test/test_step_1/test_python_api.py |  6 ++++--
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/bbot/test/test_step_1/test_command.py b/bbot/test/test_step_1/test_command.py
index f8789f8588..dd9b21f7ec 100644
--- a/bbot/test/test_step_1/test_command.py
+++ b/bbot/test/test_step_1/test_command.py
@@ -121,30 +121,30 @@ async def test_command(bbot_scanner):
     # test sudo + existence of environment variables
     await scan1._prep()
     path_parts = os.environ.get("PATH", "").split(":")
-    assert f"{BBOT_TEST_DIR}/tools" in path_parts
+    assert str(scan1.helpers.tools_dir) in path_parts
     run_lines = (await scan1.helpers.run(["env"])).stdout.splitlines()
     assert "BBOT_WEB_USER_AGENT=BBOT Test User-Agent" in run_lines
     for line in run_lines:
         if line.startswith("PATH="):
             path_parts = line.split("=", 1)[-1].split(":")
-            assert f"{BBOT_TEST_DIR}/tools" in path_parts
+            assert str(scan1.helpers.tools_dir) in path_parts
     run_lines_sudo = (await scan1.helpers.run(["env"], sudo=True)).stdout.splitlines()
     assert "BBOT_WEB_USER_AGENT=BBOT Test User-Agent" in run_lines_sudo
     for line in run_lines_sudo:
         if line.startswith("PATH="):
             path_parts = line.split("=", 1)[-1].split(":")
-            assert f"{BBOT_TEST_DIR}/tools" in path_parts
+            assert str(scan1.helpers.tools_dir) in path_parts
     run_live_lines = [l async for l in scan1.helpers.run_live(["env"])]
     assert "BBOT_WEB_USER_AGENT=BBOT Test User-Agent" in run_live_lines
     for line in run_live_lines:
         if line.startswith("PATH="):
             path_parts = line.strip().split("=", 1)[-1].split(":")
-            assert f"{BBOT_TEST_DIR}/tools" in path_parts
+            assert str(scan1.helpers.tools_dir) in path_parts
     run_live_lines_sudo = [l async for l in scan1.helpers.run_live(["env"], sudo=True)]
     assert "BBOT_WEB_USER_AGENT=BBOT Test User-Agent" in run_live_lines_sudo
     for line in run_live_lines_sudo:
         if line.startswith("PATH="):
             path_parts = line.strip().split("=", 1)[-1].split(":")
-            assert f"{BBOT_TEST_DIR}/tools" in path_parts
+            assert str(scan1.helpers.tools_dir) in path_parts
 
     await scan1._cleanup()
diff --git a/bbot/test/test_step_1/test_python_api.py b/bbot/test/test_step_1/test_python_api.py
index cb685980ea..fee826ca5e 100644
--- a/bbot/test/test_step_1/test_python_api.py
+++ b/bbot/test/test_step_1/test_python_api.py
@@ -49,7 +49,8 @@ async def test_python_api(clean_default_config):
     bbot_home = str(worker_dir("/tmp/.bbot_python_api_test"))
     scan4 = Scanner("127.0.0.1", config={"home": bbot_home})
     await scan4._prep()
-    assert os.environ["BBOT_TOOLS"] == str(Path(bbot_home) / "tools")
+    assert scan4.helpers.scans_dir == Path(bbot_home) / "scans"
+    assert os.environ["BBOT_TOOLS"] == str(scan4.helpers.tools_dir)
 
     # output modules are additive
     scan5 = Scanner()
@@ -93,7 +94,8 @@ async def test_python_api_sync(clean_default_config):
     bbot_home = str(worker_dir("/tmp/.bbot_python_api_test"))
     scan3 = Scanner("127.0.0.1", config={"home": bbot_home})
     await scan3._prep()
-    assert os.environ["BBOT_TOOLS"] == str(Path(bbot_home) / "tools")
+    assert scan3.helpers.scans_dir == Path(bbot_home) / "scans"
+    assert os.environ["BBOT_TOOLS"] == str(scan3.helpers.tools_dir)
 
 
 def test_python_api_sync_no_pending_tasks():

From 768e670cbbee078c83ebc900a031bed128e961c5 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 21:54:35 -0700
Subject: [PATCH 27/78] test(nuclei): point template paths at the shared tools
 dir

The nuclei module tests built template paths from the per-worker home,
but nuclei downloads its templates into the tools dir, which now resolves
to the shared location. The module updated templates under the shared
dir and was then handed a path under the worker dir:

    Could not find template '/tmp/.bbot_test_gw3/tools/nuclei-state/templates/...'
    Could not run nuclei: no templates provided for scan

Export BBOT_TEST_TOOLS_DIR alongside the shared dir and build the three
template paths from it, so the tests reference the same tools dir the
module installs into. The XDG env paths stay on the per-worker home,
since those intentionally point at throwaway dirs.
---
 bbot/test/test_step_2/module_tests/test_module_nuclei.py | 8 ++++----
 bbot/test/worker.py                                      | 1 +
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_nuclei.py b/bbot/test/test_step_2/module_tests/test_module_nuclei.py
index 1274a0c090..f139933e9f 100644
--- a/bbot/test/test_step_2/module_tests/test_module_nuclei.py
+++ b/bbot/test/test_step_2/module_tests/test_module_nuclei.py
@@ -1,6 +1,6 @@
 from ...bbot_fixtures import *
 from .base import ModuleTestBase
-from bbot.test.worker import HTTPSERVER_URL, BBOT_TEST_DIR
+from bbot.test.worker import HTTPSERVER_URL, BBOT_TEST_DIR, BBOT_TEST_TOOLS_DIR
 
 
 class TestNucleiManual(ModuleTestBase):
@@ -17,7 +17,7 @@ class TestNucleiManual(ModuleTestBase):
                 "mode": "manual",
                 "concurrency": 2,
                 "ratelimit": 10,
-                "templates": f"{BBOT_TEST_DIR}/tools/nuclei-state/templates/http/miscellaneous/",
+                "templates": f"{BBOT_TEST_TOOLS_DIR}/nuclei-state/templates/http/miscellaneous/",
                 "directory_only": False,
             }
         },
@@ -70,7 +70,7 @@ class TestNucleiSevere(TestNucleiManual):
             "nuclei": {
                 "mode": "severe",
                 "concurrency": 1,
-                "templates": f"{BBOT_TEST_DIR}/tools/nuclei-state/templates/http/vulnerabilities/generic/generic-env.yaml",
+                "templates": f"{BBOT_TEST_TOOLS_DIR}/nuclei-state/templates/http/vulnerabilities/generic/generic-env.yaml",
             }
         },
         "interactsh_disable": True,
@@ -116,7 +116,7 @@ class TestNucleiBudget(TestNucleiManual):
                 "mode": "budget",
                 "concurrency": 1,
                 "tags": "spiderfoot",
-                "templates": f"{BBOT_TEST_DIR}/tools/nuclei-state/templates/exposed-panels/spiderfoot.yaml",
+                "templates": f"{BBOT_TEST_TOOLS_DIR}/nuclei-state/templates/exposed-panels/spiderfoot.yaml",
             }
         },
     }
diff --git a/bbot/test/worker.py b/bbot/test/worker.py
index 0167bb1ed8..19a796fcfc 100644
--- a/bbot/test/worker.py
+++ b/bbot/test/worker.py
@@ -52,6 +52,7 @@ def worker_dir(base=BASE_BBOT_TEST_DIR):
 # worker reinstall all ~126 modules' deps from scratch. Point the install-once
 # dirs at one shared location; scan output and temp files stay per-worker.
 BBOT_TEST_SHARED_DIR = Path(f"{BASE_BBOT_TEST_DIR}_shared")
+BBOT_TEST_TOOLS_DIR = BBOT_TEST_SHARED_DIR / "tools"
 
 
 HTTPSERVER_PORT = worker_port(8888)

From e78d9bfb07d37a8cbce8573def6009c723ae9f0e Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 22:59:37 -0700
Subject: [PATCH 28/78] fix(test): check every loaded module in
 test_module_loading

Two problems, one root cause: the test called _prep() when it only ever
reads class attributes off the module instances.

The async-method check was dead. not_async was reset at the top of each
iteration of the module loop while the assert sat outside it, so only the
last module's methods were ever examined. Hoist the list out of the loop
and report the offending qualnames.

_prep() also runs setup() on every module. The ones that talk to a
service spend their whole connect-retry budget failing against a host
that isn't running: rabbitmq alone takes 29s of a hardcoded 30-attempt
loop, with postgres and mysql behind it. Worse, _prep() then drops the
failures from scan.modules, so the modules most likely to regress were
deleted before the loop could inspect them.

load_modules() instantiates every module without dialing anything, which
is all the assertions need. The test now covers 151 modules instead of
110 and takes 1.9s instead of 47.6s.
---
 bbot/test/test_step_1/test_modules_basic.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 60e4080714..62937072ee 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -524,21 +524,24 @@ async def test_module_loading(bbot_scanner):
         config={i: True for i in available_internal_modules if i != "dnsresolve"},
         force_start=True,
     )
-    await scan2._prep()
-    await scan2._set_status("RUNNING")
+    # every assertion below reads class attributes off the instantiated modules, so
+    # loading them is enough. _prep() would additionally run setup() on all ~110,
+    # and the ones that talk to a service (rabbitmq, postgres, mysql) each burn
+    # their full connect-retry budget against a service that isn't there.
+    await scan2.load_modules()
 
     # attributes, descriptions, etc.
+    not_async = []
     for module_name, module in sorted(scan2.modules.items()):
         # flags
         assert module._type in ("internal", "output", "scan")
         # async stuff
-        not_async = []
         for func_name in ("setup", "ping", "filter_event", "handle_event", "finish", "report", "cleanup"):
             f = getattr(module, func_name)
             if not scan2.helpers.is_async_function(f):
                 log.error(f"{f.__qualname__}() is not async")
-                not_async.append(f)
-    assert not any(not_async)
+                not_async.append(f.__qualname__)
+    assert not not_async, f"non-async module methods: {not_async}"
 
     await scan2._cleanup()
 

From e1fa209a0530131f8839331450747b41b4d63564 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 24 Aug 2026 23:43:28 -0700
Subject: [PATCH 29/78] fix(test): exercise the dead per_domain_only branch in
 perdomainonly

The per_domain_only assertions never ran. The loop only probed modules
watching URL, but all four per_domain_only modules (azure_tenant,
emailformat, skymem, viewdns) watch DNS_NAME, so the branch was
unreachable and its assertions had rotted since the per_host_only rework
in 0111db706:

  _per_host_tracker has not been written to since that commit; dedup
  state moved to _incoming_dup_tracker. The reason string also changed
  from "per_domain_only enabled and already seen domain" to
  "module has already seen it (per_domain_only=True)".

Probe DNS_NAME modules too so the branch is reached, and assert against
the state and reason string the implementation actually produces.
Verified negatively: the old _per_host_tracker assertion fails when the
branch is genuinely executed.

Separately, _prep() already calls setup_modules() internally, so the
explicit setup_modules() call after it re-ran setup() on every module.
The test only reads dedup state off each module, so load_modules() is
enough. Modules that dial an absent service (rabbitmq, postgres, mysql)
were each burning their full connect-retry budget, twice.

128 modules loaded vs 99 after setup_modules() pops the failures, so
coverage rises: 75 modules probed, 4 per_domain_only exercised where
previously 0. Test body drops from ~12.3s to 0.9s.
---
 bbot/test/test_step_1/test_modules_basic.py | 63 +++++++++++++--------
 1 file changed, 38 insertions(+), 25 deletions(-)

diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 62937072ee..2d34ed41eb 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -345,36 +345,49 @@ async def test_modules_basic_perdomainonly(bbot_scanner, monkeypatch):
         force_start=True,
     )
 
-    await per_domain_scan._prep()
-    await per_domain_scan.setup_modules()
+    # postcheck only reads dedup state off each module, so loading is enough.
+    # _prep() additionally runs setup() on every module, and the ones that dial a
+    # service (rabbitmq, postgres, mysql) each burn their full connect-retry budget.
+    await per_domain_scan.load_modules()
     await per_domain_scan._set_status("RUNNING")
 
-    # ensure that multiple events to the same "host" (schema + host) are blocked and check the per host tracker
-
+    # ensure that a second event under an already-seen domain is deduped away
+    per_domain_seen = []
     for module_name, module in sorted(per_domain_scan.modules.items()):
         monkeypatch.setattr(module, "filter_event", BaseModule(per_domain_scan).filter_event)
 
-        if "URL" in module.watched_events:
-            url_1 = per_domain_scan.make_event(
-                "http://www.evilcorp.com/1", event_type="URL", parent=per_domain_scan.root_event, tags=["status-200"]
-            )
-            url_1.scope_distance = 0
-            url_2 = per_domain_scan.make_event(
-                "http://mail.evilcorp.com/2", event_type="URL", parent=per_domain_scan.root_event, tags=["status-200"]
-            )
-            url_2.scope_distance = 0
-            valid_1, reason_1 = await module._event_postcheck(url_1)
-            valid_2, reason_2 = await module._event_postcheck(url_2)
-
-            if module.per_domain_only is True:
-                assert valid_1 is True
-                assert valid_2 is False
-                assert hash("evilcorp.com") in module._per_host_tracker
-                assert reason_2 == "per_domain_only enabled and already seen domain"
-
-            else:
-                assert valid_1 is True
-                assert valid_2 is True
+        # every per_domain_only module watches DNS_NAME, so probing URL-only never
+        # reached the branch below
+        if "DNS_NAME" in module.watched_events:
+            event_type, host_1, host_2 = "DNS_NAME", "www.evilcorp.com", "mail.evilcorp.com"
+        elif "URL" in module.watched_events:
+            event_type, host_1, host_2 = "URL", "http://www.evilcorp.com/1", "http://mail.evilcorp.com/2"
+        else:
+            continue
+
+        event_1 = per_domain_scan.make_event(
+            host_1, event_type=event_type, parent=per_domain_scan.root_event, tags=["status-200"]
+        )
+        event_1.scope_distance = 0
+        event_2 = per_domain_scan.make_event(
+            host_2, event_type=event_type, parent=per_domain_scan.root_event, tags=["status-200"]
+        )
+        event_2.scope_distance = 0
+        valid_1, reason_1 = await module._event_postcheck(event_1)
+        valid_2, reason_2 = await module._event_postcheck(event_2)
+
+        if module.per_domain_only is True:
+            per_domain_seen.append(module_name)
+            assert valid_1 is True, f"{module_name}: first event rejected ({reason_1})"
+            assert valid_2 is False, f"{module_name}: second event accepted"
+            assert hash("evilcorp.com") in module._incoming_dup_tracker
+            assert reason_2 == "module has already seen it (per_domain_only=True)", f"{module_name}: {reason_2}"
+
+        elif event_type == "URL":
+            assert valid_1 is True, f"{module_name}: {reason_1}"
+            assert valid_2 is True, f"{module_name}: {reason_2}"
+
+    assert per_domain_seen, "no per_domain_only modules were exercised"
 
     await per_domain_scan._cleanup()
 

From b9bdb4e25d701ec0afc4f9aec952560e46892b6c Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 00:28:50 -0700
Subject: [PATCH 30/78] perf(deps): skip the install lock when every dep is
 already satisfied

install() took an exclusive flock before deciding whether there was any
work to do. Under xdist that serializes every worker behind whichever one
is actually installing: a warm scan needing zero installs still blocked
for the full duration of an unrelated worker's install.

Visible in CI as a convoy. On py3.13 run 32818237342,
test_manager_scope_accuracy_correct was credited 241.8s while its body
runs in ~7s. The log shows gw2 stalled 06:48:53 to 06:52:54, releasing
only after gw1 finished the dep-install tests, with three workers
reporting PASSED within 20ms of each other.

Check first, outside the lock, whether the core deps cache is present and
every module is already marked installed with its pip deps satisfied. If
so, return the same (succeeded, failed) the locked path would have. The
check only reads. Anything unresolved, an unknown module, a missing cache
entry, force_install or retry_failed, still falls through to the lock, so
the install path itself is unchanged.

Readers now read setup_status.json without the lock, so write it via a
temp file plus os.replace to avoid a torn read. Hoisted the module hash
into _module_hash() so both paths compute it identically.

Measured on test_depsinstaller.py + test_modules_basic.py, -n 4
--dist loadgroup: 184s -> 8s, identical pass/fail set.
---
 bbot/core/helpers/depsinstaller/installer.py | 63 +++++++++++++++++---
 1 file changed, 55 insertions(+), 8 deletions(-)

diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index 189d10ed79..c9249ba59d 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -157,11 +157,61 @@ async def install(self, *modules):
         # Concurrent scans (notably xdist workers) share the deps dir, so serialize
         # installs across processes: without this they race on the same files and
         # each pays the full install anyway.
+        # Taking the lock unconditionally serializes every scan behind whichever one
+        # is actually installing, so check first whether there is any work to do.
+        # That check only reads, so it is safe outside the lock.
+        self.setup_status = self.read_setup_status()
+        nothing_to_do = self._all_deps_satisfied(modules)
+        if nothing_to_do is not None:
+            return nothing_to_do
         with self._install_lock():
             # another process may have installed deps while we waited for the lock
             self.setup_status = self.read_setup_status()
             return await self._install(*modules)
 
+    def _all_deps_satisfied(self, modules):
+        """Return (succeeded, failed) if every module is already installed, else None.
+
+        Mirrors the accounting in _install() so the fast path and the locked path
+        agree on which modules count as succeeded.
+        """
+        if self.deps_behavior in ("force_install", "retry_failed"):
+            return None
+        if not self._core_deps_cached():
+            return None
+        succeeded = []
+        for m in modules:
+            if self.deps_behavior == "disable":
+                succeeded.append(m)
+                continue
+            preloaded = self.all_modules_preloaded.get(m)
+            if preloaded is None:
+                return None
+            if not list(chain(*preloaded["deps"].values())):
+                succeeded.append(m)
+                continue
+            if self.setup_status.get(self._module_hash(preloaded), None) is not True:
+                return None
+            satisfied, _ = self._pip_deps_satisfied(preloaded["deps"]["pip"])
+            if not satisfied:
+                return None
+            succeeded.append(m)
+        succeeded.sort()
+        return succeeded, []
+
+    def _core_deps_cached(self):
+        core_deps_hash = str(mmh3.hash(orjson.dumps(self.CORE_DEPS, option=orjson.OPT_SORT_KEYS)))
+        return (self.parent_helper.cache_dir / core_deps_hash).exists()
+
+    def _module_hash(self, preloaded):
+        return self.parent_helper.sha1(
+            json.dumps(preloaded["deps"], sort_keys=True)
+            + self.venv
+            + str(self.parent_helper.deps_home)
+            + os.uname()[1]
+            + str(__version__)
+        ).hexdigest()
+
     @contextmanager
     def _install_lock(self):
         lock_file = self.data_dir / "install.lock"
@@ -193,13 +243,7 @@ async def _install(self, *modules):
                 log.debug(f"Installing {m} - Preloaded Deps {preloaded['deps']}")
                 # make a hash of the dependencies and check if it's already been handled
                 # take into consideration whether the venv or the deps directory changes
-                module_hash = self.parent_helper.sha1(
-                    json.dumps(preloaded["deps"], sort_keys=True)
-                    + self.venv
-                    + str(self.parent_helper.deps_home)
-                    + os.uname()[1]
-                    + str(__version__)
-                ).hexdigest()
+                module_hash = self._module_hash(preloaded)
                 success = self.setup_status.get(module_hash, None)
                 dependencies = list(chain(*preloaded["deps"].values()))
                 if len(dependencies) <= 0:
@@ -441,8 +485,11 @@ def read_setup_status(self):
         return setup_status
 
     def write_setup_status(self):
-        with open(self.setup_status_cache, "w") as f:
+        # readers take no lock, so swap the file in atomically to avoid a torn read
+        tmp = self.setup_status_cache.with_suffix(f".{os.getpid()}.tmp")
+        with open(tmp, "w") as f:
             json.dump(self.setup_status, f)
+        os.replace(tmp, self.setup_status_cache)
 
     def ensure_root(self, message=""):
         self._install_sudo_askpass()

From 914ff7cb0911c6c16a7b31526f58d28e5a6a2128 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 01:15:17 -0700
Subject: [PATCH 31/78] perf(deps): key the dep build scratch dir off
 deps_home, not home

BBOT_TEMP is substituted into module ansible playbooks by
find_and_replace(), so it lands inside preloaded["deps"], and
_module_hash() hashes that dict. Pointing it at home meant the hash
carried the per-worker home path, so the three modules whose playbooks
reference BBOT_TEMP (medusa, legba, retirejs) hashed differently on
every xdist worker. Each worker missed the shared setup_status entry
and rebuilt them from source, medusa via autoreconf + configure + make.

Measured in CI run 32821811073 (py3.13): medusa built 3 separate times
for 315.3s total, avg 105.1s per build, plus retirejs 4x/54.8s and
legba 5x/19.3s. Because install() holds the lock while building, the
other workers block on it: three of the four released within 300ms of
each other at 07:36:32 after 103s of silence, and again at 07:38:39.
That is the same convoy b9bdb4e25 fixed for the lock, arriving here
through a stale hash instead.

BBOT_TEMP is build scratch keyed on content, exactly like cache/tools/
lib, so it belongs with the other install-once dirs. Scan-scoped temp
files are unaffected: those live under Scanner.temp_dir
(home/scans/<name>/temp), which is unchanged and still per-worker.

Production is a provable no-op. deps_home returns home unless
BBOT_SHARED_DEPS_DIR is set, and only the test suite sets it.

Verified:
- medusa/legba/retirejs hash identically across two worker homes now;
  distinct=2 before, distinct=1 after. nuclei was already 1 (no
  BBOT_TEMP in its playbook), confirming the placeholder is the cause.
- End to end: seed setup_status from gw0's home, then call install()
  from gw1's home against the same shared deps dir. Fast path hits in
  0.000s, succeeded=[legba, medusa, retirejs]. Negative control on the
  pre-fix tree returns None from _all_deps_satisfied, i.e. it would
  have rebuilt medusa.
- With no BBOT_SHARED_DEPS_DIR, temp_dir is still home/temp.
- test_modules_basic + test_helpers + test_presets + test_bloom_filter:
  59 passed.
---
 bbot/core/core.py           | 4 +++-
 bbot/core/helpers/helper.py | 2 +-
 bbot/test/conftest.py       | 4 ++--
 3 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/bbot/core/core.py b/bbot/core/core.py
index dd9b9a6357..615e792047 100644
--- a/bbot/core/core.py
+++ b/bbot/core/core.py
@@ -86,7 +86,9 @@ def tools_dir(self):
 
     @property
     def temp_dir(self):
-        return self.home / "temp"
+        # BBOT_TEMP is the scratch dir for dependency builds and is baked into the
+        # module hash, so it belongs with the other install-once dirs.
+        return self.deps_home / "temp"
 
     @property
     def lib_dir(self):
diff --git a/bbot/core/helpers/helper.py b/bbot/core/helpers/helper.py
index 561cb2a933..99af85dded 100644
--- a/bbot/core/helpers/helper.py
+++ b/bbot/core/helpers/helper.py
@@ -96,7 +96,7 @@ def __init__(self, preset):
         self.cache_dir = self.deps_home / "cache"
         self.tools_dir = self.deps_home / "tools"
         self.lib_dir = self.deps_home / "lib"
-        self.temp_dir = self.bbot_home / "temp"
+        self.temp_dir = self.deps_home / "temp"
         self.scans_dir = self.bbot_home / "scans"
         self.wordlist_dir = Path(__file__).parent.parent.parent / "wordlists"
         self.current_dir = Path.cwd()
diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index 76b5f81b3b..a122c410c5 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -50,8 +50,8 @@ def thread_target(self) -> None:
 # and the sessionfinish cleanup below would delete a directory still in use.
 test_config["home"] = str(BBOT_TEST_DIR)
 
-# Deps install once into a shared dir instead of once per worker. The per-worker
-# home above still isolates scan output and temp files.
+# Deps install once into a shared dir instead of once per worker. Scan output stays
+# per-worker; the deps scratch dir does not, because it is baked into the module hash.
 os.environ.setdefault("BBOT_SHARED_DEPS_DIR", str(BBOT_TEST_SHARED_DIR))
 
 os.environ["BBOT_DEBUG"] = "True"

From aa7da78cd933f09eb4cbad33bb3dd4980d65ce08 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 02:07:10 -0700
Subject: [PATCH 32/78] perf(medusa): copy the build tree with remote_src, not
 through the controller

The "Copy medusa repo" ansible task copied a 161-file directory without
remote_src, so every file round-tripped through the ansible controller
individually. Measured locally at 114s for that single task; the whole
rest of the build (git clone, autoreconf, configure, make, make install)
is only about 20s.

medusa is the only module that builds from source, and --install-all-deps
in test_cli_args runs it while holding the global install lock. In CI
(py3.13 run 32825691146) that produced a 108s window, 08:18:54 to
08:20:42, with zero log lines from any of the 4 xdist workers. Every
worker was blocked on one directory copy.

Setting remote_src keeps the copy on the target instead of streaming each
file through the controller. Same 161 files, same working binary.

Full playbook end to end, package task excluded:
  before 143.4s, after 30.5s (4.7x), binary present and valid in both.
test_module_medusa passes (1 passed, 17.9s).
---
 bbot/modules/medusa.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bbot/modules/medusa.py b/bbot/modules/medusa.py
index 2be1333f9c..926e25a5e1 100644
--- a/bbot/modules/medusa.py
+++ b/bbot/modules/medusa.py
@@ -60,10 +60,12 @@ class Config(BaseModuleConfig):
         },
         {
             # The git repo will be copied because during build, files and subfolders get created. That prevents the Ansible git module to cache the repo.
+            # remote_src keeps the copy local; without it every file round-trips through the controller.
             "name": "Copy medusa repo",
             "copy": {
                 "src": "#{BBOT_TEMP}/medusa/gitrepo/",
                 "dest": "#{BBOT_TEMP}/medusa/workdir/",
+                "remote_src": True,
             },
         },
         {

From f34df25c370226c03a631acc89dabe1d535dcb1e Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 02:46:23 -0700
Subject: [PATCH 33/78] perf(nuclei): parse budget templates with the libyaml
 loader

NucleiBudget parses every file under nuclei-templates twice (once to
build path frequencies, once to find collapsible templates). That is
13391 yaml files, and parse_yaml used yaml.safe_load, which is the pure
python SafeLoader. Measured 58.9s of straight parsing during module
setup, all of it on the critical path before the first scan event.

pyyaml is already built with libyaml here, so CSafeLoader is available
and produces identical output. Import it as YamlLoader with a
SafeLoader fallback for builds without the C extension.

Verified:
- Deep-equality over all 13391 real templates: 0 diffs, and both loaders
  raise YAMLError on the same 0 files, so the except branch is unchanged.
- Type resolution parity checked on anchors, merge keys, unicode, block
  scalars, implicit dates, nulls, octal/hex/underscore ints, inf/nan.
- NucleiBudget end to end: 58.91s -> 8.54s (6.9x), peak RSS 537MB ->
  436MB, with byte-identical collapsible_templates, severity_stats and
  budget_paths.
- NEGATIVE CONTROL on the stashed pre-fix tree, same script: 58.91s.
- Fallback branch exercised by deleting yaml.CSafeLoader before import;
  module resolves SafeLoader and still imports clean.
- test_modules_basic 7 passed, test_helpers + test_presets +
  test_bloom_filter 52 passed. ruff check and format clean.

Found via the debug-artifact method: bucketing pytest_debug.log by
second showed a 47s window (09:22:43 -> 09:23:30) with zero output from
any worker, terminating exactly at nuclei's "Loaded [1309] templates"
line. Largest remaining dead zone in the run.
---
 bbot/modules/nuclei.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/bbot/modules/nuclei.py b/bbot/modules/nuclei.py
index a6d223770c..4e19c8e852 100644
--- a/bbot/modules/nuclei.py
+++ b/bbot/modules/nuclei.py
@@ -9,6 +9,12 @@
 from bbot.modules.base import BaseModule
 from bbot.core.config.models import BaseModuleConfig, Field
 
+try:
+    # libyaml-backed loader, ~7x faster over the full template set
+    from yaml import CSafeLoader as YamlLoader
+except ImportError:
+    from yaml import SafeLoader as YamlLoader
+
 
 class nuclei(BaseModule):
     watched_events = ["URL"]
@@ -486,7 +492,7 @@ def parse_yaml(self, yamlfile):
         if yamlfile not in self._yaml_files:
             with open(yamlfile, "r") as stream:
                 try:
-                    y = yaml.safe_load(stream)
+                    y = yaml.load(stream, Loader=YamlLoader)
                     self._yaml_files[yamlfile] = y
                 except yaml.YAMLError as e:
                     self.parent.warning(f"failed to load yaml file: {e}")

From 79c361a32b8ffd6131a40ed62f634cf1fc938ee0 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 03:23:47 -0700
Subject: [PATCH 34/78] perf(nuclei): serialize concurrent template updates on
 a lock

Every nuclei module setup() ran -update-templates unconditionally against
the shared tools dir. Under xdist all four workers hit the same 13k-file
template tree at once, each extracting over the others' work.

Measured against the real binary, four concurrent cold updates into one
shared dir: 73.4s wall. A single uncontended update: 14.5s. The extra 59s
is pure contention, and it showed up in CI as a 23.2s dead zone where no
worker logged anything, terminating exactly on "Successfully updated
nuclei templates".

Take an exclusive flock beside the templates dir. The winner updates. A
waiter that had to block wakes up, sees the tree already populated, and
skips its redundant update rather than re-running it. The lock is taken
off the event loop via run_in_executor_io so blocking on it cannot stall
the worker's loop.

Cold 4-way: 73.4s -> 15.9s, identical 13619 templates installed. Warm
4-way stays at 0.95s because the fast path never runs the subprocess.
The corruption-repair branch in setup() is unaffected: an empty templates
dir still reads as not-installed and still triggers the wipe and retry.
---
 bbot/modules/nuclei.py | 36 ++++++++++++++++++++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/bbot/modules/nuclei.py b/bbot/modules/nuclei.py
index 4e19c8e852..877ee14cfb 100644
--- a/bbot/modules/nuclei.py
+++ b/bbot/modules/nuclei.py
@@ -1,10 +1,12 @@
 import asyncio
+import fcntl
 import json
 import os
 import shutil
 import yaml
 from typing import Literal
 from itertools import islice
+from contextlib import suppress
 
 from bbot.modules.base import BaseModule
 from bbot.core.config.models import BaseModuleConfig, Field
@@ -351,6 +353,40 @@ def _nuclei_env(self):
         return env
 
     async def _update_templates(self):
+        # 13k+ template files extract into one shared dir. Concurrent updaters
+        # (xdist workers, parallel scans) fight over the same tree and each
+        # re-does the other's work, so serialize on a lock beside it. A waiter
+        # that finds the tree already populated skips its own redundant update.
+        uncontended = await self.helpers.run_in_executor_io(self._acquire_template_lock)
+        try:
+            if not uncontended and self._templates_installed():
+                self.info("Nuclei templates already up-to-date")
+                return
+            await self._run_template_update()
+        finally:
+            self._release_template_lock()
+
+    def _acquire_template_lock(self):
+        lock_path = self.helpers.tools_dir / "nuclei-templates.lock"
+        self._template_lock_file = open(lock_path, "w")
+        try:
+            fcntl.flock(self._template_lock_file, fcntl.LOCK_EX | fcntl.LOCK_NB)
+            return True
+        except OSError:
+            fcntl.flock(self._template_lock_file, fcntl.LOCK_EX)
+            return False
+
+    def _release_template_lock(self):
+        lock_file = getattr(self, "_template_lock_file", None)
+        if lock_file is None:
+            return
+        self._template_lock_file = None
+        with suppress(OSError):
+            fcntl.flock(lock_file, fcntl.LOCK_UN)
+        with suppress(OSError):
+            lock_file.close()
+
+    async def _run_template_update(self):
         self.info("Updating Nuclei templates")
         # shield so an outer cancel can't kill the subprocess mid-extract and
         # corrupt the templates dir

From 89245a182a05e322e5058f6c2e8b1a69eed356e7 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 03:57:05 -0700
Subject: [PATCH 35/78] fix(test): make docker container startup fail fast and
 self-heal

test (3.10) failed on TestKafka after burning 532s of wall time. Three
linked defects, each of which turns a transient docker error into a long
dead wait plus a hard failure on every retry.

1. Container leak on a failed attempt. kafka's check() called
   consumer.start() OUTSIDE its try block, so when the consumer could not
   reach a broker the finally block never ran and bbot-test-zookeeper
   stayed up holding port 2181. Moved consumer.start() inside the try so
   teardown always runs.

2. docker run errors were silently discarded. Every container was started
   with a fire-and-forget create_subprocess_exec whose returncode was never
   checked (mysql and postgres checked it only AFTER the port wait, which is
   too late to matter). With the leaked container still holding the name,
   attempt 2 got "Conflict. The container name /bbot-test-zookeeper is
   already in use" and attempt 3 got "failed to bind host port for
   0.0.0.0:9092: address already in use". Both were thrown away, so the test
   proceeded as if the broker had started.

3. wait_for_port_open looped forever. Having ignored the start failure, the
   test then waited on a port nothing would ever bind. The log shows 1051
   consecutive "Waiting for port 9092" lines at 0.5s each, roughly 532s of a
   14m11s job spent spinning on a container that was never running.

Adds start_container/stop_container on ModuleTestBase. start_container does
docker rm -f first so a leaked container from a prior attempt cannot poison
the retry, and raises with docker's stderr when the run fails. Bounds
wait_for_port_open by the existing CONTAINER_READY_TIMEOUT (180s) and reuses
the OOM hint. stop_container uses rm -f rather than stop, so teardown removes
the name even when the container is already dead.

Also drops the mysql/postgres teardown exceptions that raised before the
sibling container was stopped, which was another way to leak a container.
Applied the helpers to all seven docker module tests: kafka, mongo, mysql,
nats, postgres, rabbitmq, elastic. No assertions changed.

Verified against real docker:
- start over a live container of the same name recovers in 0.6s
- bad image raises in 1.3s with docker's stderr instead of hanging
- wait_for_port_open(timeout=3) raises at 3.0s
- stop_container removes the container; a no-op on an absent name
- NEGATIVE CONTROL: faithful pre-fix reimplementation with a leaked
  container still spinning at the 12s cap, unbounded in CI. This is the
  proof the change is load-bearing.
- mysql/postgres/social fail identically on the stashed clean tree, local
  ports 3306/5432 are held by unrelated containers. Not regressions.
- ruff check + ruff format --check clean.
---
 bbot/test/test_step_2/module_tests/base.py    | 55 ++++++++++++++++++-
 .../module_tests/test_module_elastic.py       | 13 +----
 .../module_tests/test_module_kafka.py         | 26 +++------
 .../module_tests/test_module_mongo.py         | 14 +----
 .../module_tests/test_module_mysql.py         | 23 +-------
 .../module_tests/test_module_nats.py          | 10 +---
 .../module_tests/test_module_postgres.py      | 20 +------
 .../module_tests/test_module_rabbitmq.py      | 21 +------
 8 files changed, 74 insertions(+), 108 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index 2c55ad6724..d6578d37f5 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -1,3 +1,4 @@
+import time
 import pytest
 import asyncio
 import logging
@@ -7,6 +8,7 @@
 from bbot.scanner import Scanner
 from bbot.core.config.merge import deep_merge
 from bbot.core.helpers.misc import rand_string
+from bbot.test.worker import CONTAINER_READY_TIMEOUT, _OOM_HINT
 
 log = logging.getLogger("bbot.test.modules")
 
@@ -178,13 +180,64 @@ async def setup_after_prep(self, module_test):
     async def _mock_http_wildcard(*args, **kwargs):
         return False
 
-    async def wait_for_port_open(self, port):
+    async def wait_for_port_open(self, port, timeout=CONTAINER_READY_TIMEOUT):
+        deadline = time.time() + timeout
         while not await self.is_port_open("localhost", port):
+            if time.time() > deadline:
+                raise RuntimeError(
+                    f"Port {port} did not open within {timeout}s, so the container never came up. {_OOM_HINT}"
+                )
             self.log.verbose(f"Waiting for port {port} to be open...")
             await asyncio.sleep(0.5)
         # allow an extra second for things to settle
         await asyncio.sleep(1)
 
+    async def start_container(self, name, *args):
+        """Start a detached container, replacing any leftover of the same name.
+
+        A container the previous attempt failed to remove keeps its published
+        ports bound, so the retry's ``docker run`` fails and the test then waits
+        on a port nothing will ever listen on.
+        """
+        rm = await asyncio.create_subprocess_exec(
+            "docker",
+            "rm",
+            "-f",
+            name,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        await rm.communicate()
+
+        proc = await asyncio.create_subprocess_exec(
+            "docker",
+            "run",
+            "-d",
+            "--rm",
+            "--name",
+            name,
+            *args,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        stdout, stderr = await proc.communicate()
+        if proc.returncode != 0:
+            raise RuntimeError(
+                f"Failed to start container {name} (exit {proc.returncode}): {stderr.decode(errors='replace').strip()}"
+            )
+        return stdout.decode(errors="replace").strip()
+
+    async def stop_container(self, name):
+        proc = await asyncio.create_subprocess_exec(
+            "docker",
+            "rm",
+            "-f",
+            name,
+            stdout=asyncio.subprocess.PIPE,
+            stderr=asyncio.subprocess.PIPE,
+        )
+        await proc.communicate()
+
     async def is_port_open(self, host, port):
         try:
             reader, writer = await asyncio.open_connection(host, port)
diff --git a/bbot/test/test_step_2/module_tests/test_module_elastic.py b/bbot/test/test_step_2/module_tests/test_module_elastic.py
index 95cf083adc..1b91210c28 100644
--- a/bbot/test/test_step_2/module_tests/test_module_elastic.py
+++ b/bbot/test/test_step_2/module_tests/test_module_elastic.py
@@ -1,5 +1,4 @@
 import json
-import asyncio
 import ssl
 from urllib.request import urlopen, Request
 from urllib.error import URLError
@@ -37,12 +36,8 @@ class TestElastic(ModuleTestBase):
 
     async def setup_before_prep(self, module_test):
         # Start Elasticsearch container
-        await asyncio.create_subprocess_exec(
-            "docker",
-            "run",
-            "--name",
+        await self.start_container(
             "bbot-test-elastic",
-            "--rm",
             "-e",
             "ELASTIC_PASSWORD=bbotislife",
             "-e",
@@ -53,7 +48,6 @@ async def setup_before_prep(self, module_test):
             "cluster.routing.allocation.disk.watermark.flood_stage=98%",
             "-p",
             "9200:9200",
-            "-d",
             "docker.elastic.co/elasticsearch/elasticsearch:8.16.0",
         )
 
@@ -123,7 +117,4 @@ async def check(self, module_test, events):
             except URLError:
                 pass
             self.log.verbose("Deleted documents from index")
-            process = await asyncio.create_subprocess_exec(
-                "docker", "stop", "bbot-test-elastic", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            await process.communicate()
+            await self.stop_container("bbot-test-elastic")
diff --git a/bbot/test/test_step_2/module_tests/test_module_kafka.py b/bbot/test/test_step_2/module_tests/test_module_kafka.py
index 451a551e5f..2822981566 100644
--- a/bbot/test/test_step_2/module_tests/test_module_kafka.py
+++ b/bbot/test/test_step_2/module_tests/test_module_kafka.py
@@ -17,20 +17,13 @@ class TestKafka(ModuleTestBase):
 
     async def setup_before_prep(self, module_test):
         # Start Zookeeper
-        await asyncio.create_subprocess_exec(
-            "docker", "run", "-d", "--rm", "--name", "bbot-test-zookeeper", "-p", "2181:2181", "zookeeper:3.9"
-        )
+        await self.start_container("bbot-test-zookeeper", "-p", "2181:2181", "zookeeper:3.9")
 
         # Wait for Zookeeper to be ready
         await self.wait_for_port_open(2181)
 
         # Start Kafka using wurstmeister/kafka
-        await asyncio.create_subprocess_exec(
-            "docker",
-            "run",
-            "-d",
-            "--rm",
-            "--name",
+        await self.start_container(
             "bbot-test-kafka",
             "--link",
             "bbot-test-zookeeper:zookeeper",
@@ -61,9 +54,12 @@ async def check(self, module_test, events):
             group_id="test_group",
             auto_offset_reset="earliest",
         )
-        await self.consumer.start()
 
         try:
+            # inside the try: a failure here must still tear the containers down,
+            # otherwise they hold port 9092 and every retry fails to bind it
+            await self.consumer.start()
+
             events_json = [e.json() for e in events]
             events_json.sort(key=lambda x: x["timestamp"])
 
@@ -89,11 +85,5 @@ async def _consume():
             if hasattr(self, "consumer") and not self.consumer._closed:
                 await self.consumer.stop()
             # Stop Kafka and Zookeeper containers
-            p1 = await asyncio.create_subprocess_exec(
-                "docker", "stop", "bbot-test-kafka", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            await p1.communicate()
-            p2 = await asyncio.create_subprocess_exec(
-                "docker", "stop", "bbot-test-zookeeper", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            await p2.communicate()
+            await self.stop_container("bbot-test-kafka")
+            await self.stop_container("bbot-test-zookeeper")
diff --git a/bbot/test/test_step_2/module_tests/test_module_mongo.py b/bbot/test/test_step_2/module_tests/test_module_mongo.py
index f889dc1204..5e3262141c 100644
--- a/bbot/test/test_step_2/module_tests/test_module_mongo.py
+++ b/bbot/test/test_step_2/module_tests/test_module_mongo.py
@@ -1,5 +1,3 @@
-import asyncio
-
 from .base import ModuleTestBase
 from bbot.test.worker import wait_for_container
 
@@ -20,19 +18,14 @@ class TestMongo(ModuleTestBase):
     skip_distro_tests = True
 
     async def setup_before_prep(self, module_test):
-        await asyncio.create_subprocess_exec(
-            "docker",
-            "run",
-            "--name",
+        await self.start_container(
             "bbot-test-mongo",
-            "--rm",
             "-e",
             "MONGO_INITDB_ROOT_USERNAME=bbot",
             "-e",
             "MONGO_INITDB_ROOT_PASSWORD=bbotislife",
             "-p",
             "27017:27017",
-            "-d",
             "mongo",
         )
         # Wait for port to be available
@@ -147,7 +140,4 @@ async def check(self, module_test, events):
             await events_collection.delete_many({})
             # Close the MongoDB connection
             await client.aclose()
-            process = await asyncio.create_subprocess_exec(
-                "docker", "stop", "bbot-test-mongo", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            await process.communicate()
+            await self.stop_container("bbot-test-mongo")
diff --git a/bbot/test/test_step_2/module_tests/test_module_mysql.py b/bbot/test/test_step_2/module_tests/test_module_mysql.py
index de30c58f9f..77a34cee0c 100644
--- a/bbot/test/test_step_2/module_tests/test_module_mysql.py
+++ b/bbot/test/test_step_2/module_tests/test_module_mysql.py
@@ -1,5 +1,3 @@
-import asyncio
-
 from .base import ModuleTestBase
 
 
@@ -8,31 +6,20 @@ class TestMySQL(ModuleTestBase):
     skip_distro_tests = True
 
     async def setup_before_prep(self, module_test):
-        process = await asyncio.create_subprocess_exec(
-            "docker",
-            "run",
-            "--name",
+        await self.start_container(
             "bbot-test-mysql",
-            "--rm",
             "-e",
             "MYSQL_ROOT_PASSWORD=bbotislife",
             "-e",
             "MYSQL_DATABASE=bbot",
             "-p",
             "3306:3306",
-            "-d",
             "mysql",
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
         )
-        stdout, stderr = await process.communicate()
 
         # wait for the container to start
         await self.wait_for_port_open(3306)
 
-        if process.returncode != 0:
-            self.log.error(f"Failed to start MySQL server: {stderr.decode()}")
-
     async def check(self, module_test, events):
         import aiomysql
 
@@ -54,10 +41,4 @@ async def check(self, module_test, events):
                 assert len(targets) == 1, "No targets found in MySQL database"
         finally:
             conn.close()
-            process = await asyncio.create_subprocess_exec(
-                "docker", "stop", "bbot-test-mysql", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            stdout, stderr = await process.communicate()
-
-            if process.returncode != 0:
-                raise Exception(f"Failed to stop MySQL server: {stderr.decode()}")
+            await self.stop_container("bbot-test-mysql")
diff --git a/bbot/test/test_step_2/module_tests/test_module_nats.py b/bbot/test/test_step_2/module_tests/test_module_nats.py
index 41ae021cac..9d3e8fba68 100644
--- a/bbot/test/test_step_2/module_tests/test_module_nats.py
+++ b/bbot/test/test_step_2/module_tests/test_module_nats.py
@@ -1,5 +1,4 @@
 import json
-import asyncio
 from contextlib import suppress
 
 from .base import ModuleTestBase
@@ -18,9 +17,7 @@ class TestNats(ModuleTestBase):
 
     async def setup_before_prep(self, module_test):
         # Start NATS server
-        await asyncio.create_subprocess_exec(
-            "docker", "run", "-d", "--rm", "--name", "bbot-test-nats", "-p", "4222:4222", "nats:latest"
-        )
+        await self.start_container("bbot-test-nats", "-p", "4222:4222", "nats:latest")
 
         # Wait for NATS to be ready by checking the port
         await self.wait_for_port_open(4222)
@@ -61,7 +58,4 @@ async def check(self, module_test, events):
                     await self.nc.drain()
                 await self.nc.close()
             # Stop NATS server container
-            process = await asyncio.create_subprocess_exec(
-                "docker", "stop", "bbot-test-nats", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            await process.communicate()
+            await self.stop_container("bbot-test-nats")
diff --git a/bbot/test/test_step_2/module_tests/test_module_postgres.py b/bbot/test/test_step_2/module_tests/test_module_postgres.py
index 8c52eabebe..e3d1e47457 100644
--- a/bbot/test/test_step_2/module_tests/test_module_postgres.py
+++ b/bbot/test/test_step_2/module_tests/test_module_postgres.py
@@ -1,5 +1,3 @@
-import asyncio
-
 from .base import ModuleTestBase
 
 
@@ -8,28 +6,20 @@ class TestPostgres(ModuleTestBase):
     skip_distro_tests = True
 
     async def setup_before_prep(self, module_test):
-        process = await asyncio.create_subprocess_exec(
-            "docker",
-            "run",
-            "--name",
+        await self.start_container(
             "bbot-test-postgres",
-            "--rm",
             "-e",
             "POSTGRES_PASSWORD=bbotislife",
             "-e",
             "POSTGRES_USER=postgres",
             "-p",
             "5432:5432",
-            "-d",
             "postgres",
         )
 
         # wait for the container to start
         await self.wait_for_port_open(5432)
 
-        if process.returncode != 0:
-            self.log.error("Failed to start PostgreSQL server")
-
     async def check(self, module_test, events):
         import asyncpg
 
@@ -45,10 +35,4 @@ async def check(self, module_test, events):
             assert len(targets) == 1, "No targets found in PostgreSQL database"
         finally:
             await conn.close()
-            process = await asyncio.create_subprocess_exec(
-                "docker", "stop", "bbot-test-postgres", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            stdout, stderr = await process.communicate()
-
-            if process.returncode != 0:
-                raise Exception(f"Failed to stop PostgreSQL server: {stderr.decode()}")
+            await self.stop_container("bbot-test-postgres")
diff --git a/bbot/test/test_step_2/module_tests/test_module_rabbitmq.py b/bbot/test/test_step_2/module_tests/test_module_rabbitmq.py
index 0e76fd05db..0ec01ec307 100644
--- a/bbot/test/test_step_2/module_tests/test_module_rabbitmq.py
+++ b/bbot/test/test_step_2/module_tests/test_module_rabbitmq.py
@@ -1,5 +1,4 @@
 import json
-import asyncio
 
 from .base import ModuleTestBase
 
@@ -16,21 +15,8 @@ class TestRabbitMQ(ModuleTestBase):
     skip_distro_tests = True
 
     async def setup_before_prep(self, module_test):
-        # Remove any leftover container from a previous failed run
-        proc = await asyncio.create_subprocess_exec(
-            "docker",
-            "rm",
-            "-f",
-            "bbot-test-rabbitmq",
-            stdout=asyncio.subprocess.PIPE,
-            stderr=asyncio.subprocess.PIPE,
-        )
-        await proc.wait()
-
         # Start RabbitMQ
-        await asyncio.create_subprocess_exec(
-            "docker", "run", "-d", "--rm", "--name", "bbot-test-rabbitmq", "-p", "5672:5672", "rabbitmq:3-management"
-        )
+        await self.start_container("bbot-test-rabbitmq", "-p", "5672:5672", "rabbitmq:3-management")
 
         # Wait for RabbitMQ to be ready by checking the port
         await self.wait_for_port_open(5672)
@@ -65,7 +51,4 @@ async def check(self, module_test, events):
             # Clean up: Close the RabbitMQ connection
             await connection.close()
             # Stop RabbitMQ container
-            process = await asyncio.create_subprocess_exec(
-                "docker", "stop", "bbot-test-rabbitmq", stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
-            )
-            await process.communicate()
+            await self.stop_container("bbot-test-rabbitmq")

From 19dbdf7799bfa0f9ceb98bb232671e304553fad2 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 04:25:14 -0700
Subject: [PATCH 36/78] perf(deps): skip the community.general install when
 ansible-core suffices

install_core_deps() unconditionally shelled out to `ansible-galaxy collection
install community.general` on every cold config dir. On the CI runners that is
15.6s of download for a collection nothing in the run ever loads.

Measured on py3.10 run 32839809554 (job 97776672473), pytest_debug.log:
  10:58:12,758 Installing Ansible Community General Collection
  10:58:28,373 Successfully installed
The scan that paid it had "No scan modules to load" and every module reporting
"No dependency work to do". Pure dead wall time.

Root cause: the collection is only load-bearing because ansible's `package`
action resolves to a per-manager module. ansible-core bundles apt, dnf and dnf5;
pacman, apk, zypper, portage and friends live in community.general. CI runs on
ubuntu-latest, which is apt, so the dispatch target was always already present.
Every ansible module bbot actually names in its task definitions (command, copy,
file, get_url, git, package, shell, unarchive, uri) is bundled in ansible-core.

Fix: probe which package manager binaries exist via ansible's own PKG_MGRS table
and skip the galaxy install only when every one of them is bundled. Hosts that
need it still install it, so non-Debian users are unaffected.

Verified:
- dispatch, with the collection unavailable: apt and dnf resolve, pacman and
  zypper fail with 'Could not find a matching action for the "pacman" package
  manager'. Confirms both the necessity and the exemption.
- platform matrix: ubuntu/apt True, fedora/dnf True, arch/pacman False,
  alpine/apk False, suse/zypper False, macos/brew False, no-pkg-mgr False,
  and ubuntu with a stray pacman False (any unbundled manager vetoes).
- this Arch box correctly returns False and keeps installing.
- fails safe: ansible internals unimportable returns False.
- cost 69ms cold, 0.1ms warm, against the 15.6s it removes.
- ruff check + ruff format --check clean; test_modules_basic, test_presets,
  test_python_api pass. test_depsinstaller fails identically on the stashed
  clean tree (known local sudo/ansible env issue), not a regression.
---
 bbot/core/helpers/depsinstaller/installer.py | 45 +++++++++++++++-----
 1 file changed, 35 insertions(+), 10 deletions(-)

diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index c9249ba59d..bb320bf992 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -631,17 +631,22 @@ async def install_core_deps(self):
         # install ansible community.general collection if needed
         overall_success = True
         if not self.setup_status.get("ansible:community.general", False):
-            log.info("Installing Ansible Community General Collection")
-            try:
-                command = ["ansible-galaxy", "collection", "install", "community.general"]
-                await self.parent_helper.run(command, check=True)
-                self.setup_status["ansible:community.general"] = True
-                log.info("Successfully installed Ansible Community General Collection")
-            except CalledProcessError as err:
-                log.warning(
-                    f"Failed to install Ansible Community.General Collection (return code {err.returncode}): {err.stderr}"
+            if self._local_pkg_mgrs_are_builtin():
+                log.debug(
+                    "Skipping Ansible Community General Collection (local package manager is built into ansible-core)"
                 )
-                overall_success = False
+            else:
+                log.info("Installing Ansible Community General Collection")
+                try:
+                    command = ["ansible-galaxy", "collection", "install", "community.general"]
+                    await self.parent_helper.run(command, check=True)
+                    self.setup_status["ansible:community.general"] = True
+                    log.info("Successfully installed Ansible Community General Collection")
+                except CalledProcessError as err:
+                    log.warning(
+                        f"Failed to install Ansible Community.General Collection (return code {err.returncode}): {err.stderr}"
+                    )
+                    overall_success = False
         # only run ansible if there's actually something to install
         if playbook:
             self._install_sudo_askpass()
@@ -658,6 +663,26 @@ async def install_core_deps(self):
             with suppress(Exception):
                 core_deps_cache_file.touch()
 
+    @staticmethod
+    def _local_pkg_mgrs_are_builtin():
+        """True if every package manager present on this host ships with ansible-core.
+
+        The only reason we install community.general is that the `package` action
+        dispatches to a per-manager module (pacman, apk, zypper...) that lives in
+        that collection. ansible-core bundles apt/dnf/dnf5, so on those hosts the
+        galaxy install is a pure no-op download.
+        """
+        try:
+            import ansible.modules
+            from ansible.module_utils.facts.system.pkg_mgr import PKG_MGRS
+        except Exception:
+            return False
+        bundled = Path(ansible.modules.__file__).resolve().parent
+        present = {p["name"] for p in PKG_MGRS if os.path.exists(p["path"])}
+        if not present:
+            return False
+        return all((bundled / f"{name}.py").is_file() for name in present)
+
     def _setup_sudo_cache(self):
         if not self._sudo_cache_setup:
             self._sudo_cache_setup = True

From e393514afe5e7841b4f89bf85ba2ed729b08f1c8 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 04:52:18 -0700
Subject: [PATCH 37/78] perf(test): stop forcing a full dep reinstall in the
 gowitness tests

TestGowitness and TestGowitness_MultiPort set deps.behavior=force_install,
which makes every scan they run reinstall gowitness from scratch: the
11-task shared chromium playbook plus the gowitness binary download. Three
of the five tests in the file inherit that setting, so CI paid it four
separate times.

The flag dates back to 2023 (36acee410), when tools_dir was derived from
bbot_home. The tests set a private home and rmtree it at import, so back
then the wipe really did remove the installed tools and force_install was
load-bearing. It is not anymore. tools_dir, cache_dir and temp_dir now come
from BBOTCore.deps_home, which the test suite points at a shared dir via
BBOT_SHARED_DEPS_DIR; only scans_dir still follows home. So the rmtree
clears scan output and leaves the tools completely untouched, and
force_install now reinstalls tools that were never removed.

It was also actively harmful beyond its own runtime. force_install bypasses
the already-installed check for shared deps (installer.py:322), so these
tests rewrote the shared chromium setup_status entry on every run.

Removing it costs no coverage: TestGoWitnessWithBlob and
TestGoWitnessLongFilename already run without force_install and pass in CI,
exercising the same module against the same shared tools dir.

Measured on py3.10 run 32839809554, gowitness ran the chromium playbook 4x
plus 4 binary downloads, 26.6s of the 107.7s total ansible time. In-CI test
durations show the split cleanly: the three force_install tests took 9.9s,
14.3s and 14.8s while the two without it took 6.2s and 4.6s.

Verified:
- Structurally: tools_dir/cache_dir resolve under deps_home, not under the
  test's home, so rmtree(home_dir) cannot remove them.
- Cold runner: with an empty shared deps dir and no setup_status.json, the
  fast path returns None and still falls through to a real install.
- Warm runner: returns (['gowitness'], []) and skips the playbook.
- Guard intact: a missing pip dep still declines the fast path.
- gowitness is the only deps_common=chromium consumer, so no sibling module
  depended on the forced refresh.
- Local setup time 8.79s -> 0.53s per test, same 5 pre-existing chromium
  errors before and after (chromium needs sudo to install on this box).
- ruff check and ruff format clean. test_depsinstaller fails identically on
  the stashed clean tree, confirmed by stash and rerun.
---
 bbot/test/test_step_2/module_tests/test_module_gowitness.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_gowitness.py b/bbot/test/test_step_2/module_tests/test_module_gowitness.py
index 8df6f9ae39..e5c74e2e39 100644
--- a/bbot/test/test_step_2/module_tests/test_module_gowitness.py
+++ b/bbot/test/test_step_2/module_tests/test_module_gowitness.py
@@ -20,7 +20,6 @@ class TestGowitness(ModuleTestBase):
     home_dir = worker_dir("/tmp/.bbot_gowitness_test")
     shutil.rmtree(home_dir, ignore_errors=True)
     config_overrides = {
-        "deps": {"behavior": "force_install"},
         "home": str(home_dir),
         "scope": {"report_distance": 2},
         "omit_event_types": [],
@@ -158,7 +157,6 @@ class TestGowitness_MultiPort(ModuleTestBase):
     home_dir = worker_dir("/tmp/.bbot_gowitness_multiport_test")
     shutil.rmtree(home_dir, ignore_errors=True)
     config_overrides = {
-        "deps": {"behavior": "force_install"},
         "home": str(home_dir),
         "omit_event_types": [],
     }

From 09145145b8f4fe776fba7f4dc69ebf11932134e8 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 05:22:13 -0700
Subject: [PATCH 38/78] perf(deps): cache ansible facts across playbook runs

Every ansible_run() re-ran the "Gathering Facts" step from scratch. That
step costs ~1.6s and runs before any task, so a 1-task playbook that does
almost nothing still took ~2.5s. A CI job makes ~16 ansible_run() calls,
so the suite paid the fact-gathering toll ~16 times for facts that never
change within a run.

The facts are needed: shared_deps.py and installer.py gate 30+ tasks on
ansible_facts os_family/distribution/system/architecture, so gather_facts
cannot simply be turned off. Caching them is the correct fix.

ansible_runner forces ANSIBLE_CACHE_PLUGIN_CONNECTION to
<artifact_dir>/<ident>/fact_cache, and bbot rmtree'd its data dir on every
call, so the cache could never survive. Point artifact_dir at a stable
shared dir and use the settings fact_cache key (which is joined onto
artifact_dir) to hop up one level, so the cache lands beside the per-run
idents rather than inside one. ident stays a fresh uuid4 so each run's
events remain isolated; sharing it replays stale events and would corrupt
failure detection. The per-run artifact dir is removed after res.events is
consumed, since events are read lazily off disk.

A corrupt cache entry would otherwise make every subsequent playbook fail
until the 24h timeout expired, so unreadable entries are discarded at
construction.

Measured, real ansible_run() on conditional tasks: 2.5s -> 0.9s warm,
cold path unchanged. Verified skip/ok conditionals resolve identically
before and after, failing playbooks still report status and the same error
string, 4 concurrent cold writers all succeed (jsonfile cache writes are
atomic via mkstemp+rename), expired entries are re-gathered, and a missing
or corrupt cache falls back to a full gather.
---
 bbot/core/helpers/depsinstaller/installer.py | 29 ++++++++++++++++++++
 1 file changed, 29 insertions(+)

diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index bb320bf992..259c156bcb 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -8,6 +8,7 @@
 import shutil
 import getpass
 import logging
+import uuid
 from time import sleep
 from pathlib import Path
 from threading import Lock
@@ -137,6 +138,10 @@ def __init__(self, parent_helper):
         self.setup_status_cache = self.data_dir / "setup_status.json"
         self.command_status = self.data_dir / "command_status"
         self.parent_helper.mkdir(self.command_status)
+        self.ansible_artifact_dir = self.data_dir / "ansible_artifacts"
+        self.parent_helper.mkdir(self.ansible_artifact_dir)
+        self.ansible_fact_cache = self.ansible_artifact_dir / "fact_cache"
+        self._discard_corrupt_fact_cache()
         self.setup_status = self.read_setup_status()
 
         # make sure we're using a minimal git config
@@ -423,6 +428,20 @@ def tasks(self, module, tasks):
             log.error(f"Failed to run Ansible tasks for {module}")
         return success
 
+    def _discard_corrupt_fact_cache(self):
+        # an unreadable entry makes every later playbook fail until it expires
+        if not self.ansible_fact_cache.is_dir():
+            return
+        for entry in self.ansible_fact_cache.iterdir():
+            if not entry.is_file():
+                continue
+            try:
+                json.loads(entry.read_text())
+            except Exception:
+                log.debug(f"Discarding corrupt ansible fact cache entry: {entry}")
+                with suppress(OSError):
+                    entry.unlink()
+
     def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
         _ansible_args = {"ansible_connection": "local", "ansible_python_interpreter": sys.executable}
         if ansible_args is not None:
@@ -450,9 +469,17 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
         shutil.rmtree(data_dir, ignore_errors=True)
         self.parent_helper.mkdir(data_dir)
 
+        # unique ident keeps each run's events isolated; fact_cache escapes it so
+        # facts are gathered once instead of on every playbook
+        ident = uuid.uuid4().hex
+
         res = run(
             playbook=playbook,
             private_data_dir=str(data_dir),
+            artifact_dir=str(self.ansible_artifact_dir),
+            ident=ident,
+            settings={"fact_cache": "../fact_cache", "fact_cache_type": "jsonfile"},
+            envvars={"ANSIBLE_GATHERING": "smart", "ANSIBLE_CACHE_PLUGIN_TIMEOUT": "86400"},
             host_pattern="localhost",
             inventory={
                 "all": {"hosts": {"localhost": _ansible_args}},
@@ -474,6 +501,8 @@ def ansible_run(self, tasks=None, module=None, args=None, ansible_args=None):
             if e["event"] == "runner_on_failed":
                 err = e["event_data"]["res"]["msg"]
                 break
+        # events are read lazily out of the artifact dir, so only discard it once they are consumed
+        shutil.rmtree(self.ansible_artifact_dir / ident, ignore_errors=True)
         return success, err
 
     def read_setup_status(self):

From 04589f452da23abb755bf4a375be211ead894d78 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 06:05:22 -0700
Subject: [PATCH 39/78] fix(test): wait for the container service, not the
 published port

test (3.11) failed TestKafka with "Unable to bootstrap from
[('localhost', 9092)]" and test (3.14) errored TestMySQL with "Lost
connection to MySQL server during query" after 11 attempts. Both are the
same defect, exposed by 89245a182 rather than caused by it.

ROOT CAUSE: docker binds a published host port at container CREATE time,
before the containerized service is listening. Connections that arrive in
that window are accepted by the docker proxy and immediately EOF'd.
is_port_open() did a bare connect and returned True on success, so
wait_for_port_open() was satisfied by the proxy alone and returned while
the broker/server was still booting.

Before 89245a182 the `docker run` subprocess was not awaited, so the
port did not exist yet on the first probe and the accidental 1.5s of
sleep(0.5) loop plus the trailing sleep(1) usually covered zookeeper's
boot. Awaiting `docker run` removed that accidental delay and the race
went from usually-hidden to usually-lost. Measured on this box:

  PRE-FIX  (docker run NOT awaited): ready at 1.51s, loops=1
  POST-FIX (docker run awaited)    : ready at 1.38s, loops=0

Zero loop iterations means the port check never actually waited for
anything. This also explains why "Waiting for port" appears 0 times in
every job log, passing and failing alike, and why the kafka retries in
the 3.11 log took only 1.3s and 3.1s: far too fast for Kafka to boot.

FIX: treat an instant EOF as not-yet-listening. Only a connection that
stays open past a short settle window, or one that sends a banner, counts
as ready. Callers and the CONTAINER_READY_TIMEOUT bound are unchanged, so
a service that never comes up still fails fast with the same error.

Verified against real docker:
- Time-to-ready now tracks real service boot instead of port binding:
  zookeeper 1.0s -> 2.5s, nats 1.0s -> 2.0s, rabbitmq 1.0s -> 9.6s,
  mysql 1.0s -> 16.1s.
- DECISIVE NEGATIVE CONTROL on mysql, handshake attempted at the instant
  each helper declares ready:
    OLD: ready at  1.01s -> read returns b'' (server not up, module fails)
    NEW: ready at 18.10s -> real MySQL greeting
- Guards: dead port raises at the timeout bound (3.02s of 3s); a
  published-but-silent container correctly raises instead of being
  declared ready (the old bug); a real listener that sends no banner is
  still detected at 1.50s, so no false negative.
- kafka 1 passed in 79.7s, mysql 1 passed in 21.0s, both previously
  failing in CI. kafka/nats/rabbitmq/mysql/elastic: 5 passed.
- mongo errors locally on "port is already allocated", 27017 is held by
  an unrelated bbot-backend container. Fails identically on the stashed
  clean tree. Not a regression.
- ruff check + ruff format --check clean.

No assertions changed, no tests split.
---
 bbot/test/test_step_2/module_tests/base.py | 24 ++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index d6578d37f5..b2b27dd49e 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -3,6 +3,7 @@
 import asyncio
 import logging
 import pytest_asyncio
+from contextlib import suppress
 
 from ...bbot_fixtures import *
 from bbot.scanner import Scanner
@@ -238,11 +239,26 @@ async def stop_container(self, name):
         )
         await proc.communicate()
 
-    async def is_port_open(self, host, port):
+    async def is_port_open(self, host, port, settle=0.5):
+        """Return True only once something is really listening behind ``port``.
+
+        docker publishes a port by binding it on the host at container create
+        time, so a bare connect succeeds while the containerized service is
+        still booting. The proxy then immediately EOFs that connection. Treat an
+        instant EOF as not-yet-listening; a connection that stays open, or one
+        that sends a banner, means the service is up.
+        """
+        writer = None
         try:
             reader, writer = await asyncio.open_connection(host, port)
-            writer.close()
-            await writer.wait_closed()
-            return True
+            try:
+                return await asyncio.wait_for(reader.read(1), timeout=settle) != b""
+            except asyncio.TimeoutError:
+                return True
         except (ConnectionRefusedError, OSError):
             return False
+        finally:
+            if writer is not None:
+                writer.close()
+                with suppress(ConnectionResetError, OSError):
+                    await writer.wait_closed()

From c1768d8a2533f762ade6f3d7c53befd671f4c4cc Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 06:55:14 -0700
Subject: [PATCH 40/78] perf(diff): skip DeepDiff pair-matching for flat string
 bodies

HttpCompare._baseline() diffs two baseline responses with ignore_order=True.
When the body is not XML, both sides are text.split("\n"), so a page with many
dynamic lines hands DeepDiff a large set of unmatched string items. DeepDiff
then runs _get_most_in_common_pairs_in_iterables, which constructs a nested
DeepDiff for every added/removed combination to compute a rough distance. For
600 differing lines that is 360000 nested diffs: 28.8s in one call, holding the
GIL.

This is the same quadratic pairing stall that dd91685f7 bounded in
compare_body for issue #3339, but _baseline() was never given the same
protection and runs before any threshold can apply.

Pair-matching only exists to align items that can be diffed deeper. When every
item is a string there is no substructure to align, and DeepDiff's
TreeResult.mutual_add_removes_to_become_value_changes already folds equal-index
add/remove pairs back into values_changed. So for flat string sequences the
pairing pass is pure cost with no effect on the result.

_ordered_diff() centralizes the three ignore_order call sites and sets
cutoff_intersection_for_pairs=0 only when both sides are flat all-string lists.
Dict bodies (xmltodict output) and any mixed-type list keep the full pairing
path, since there the pass does change path granularity.

Verified:
- 2650 randomized and exhaustive flat-string cases compared on the full payload
  (report key, path, t1, t2, report_type): 0 mismatches. 8045 cases compared on
  paths and key counts: 0 mismatches.
- Confirmed the pass is genuinely skipped by spying on
  _get_most_in_common_pairs_in_iterables: 0 calls for all-string input, still 1
  call for dict input and for mixed-type lists.
- Guard checked against lists, tuples, dicts, bytes, None, nested and unicode.
  Only flat all-string lists take the fast path.
- The 600-line baseline diff goes 24.4s -> 0.06s (380x) with byte-identical
  tree output. Pre-fix timing on the same input is the negative control.
- test_web.py: 52.35s -> 26.67s, with an identical 3 failed / 18 passed set
  before and after. Those 3 fail on the clean tree too (stale local Rust
  extension, no decode_error attribute).
- test_helpers/test_presets/test_bloom_filter/test_scan: 65 passed.
- url_manipulation, bypass403, paramminer_headers: identical 9 failed / 3
  passed before and after, all pre-existing locally.

This is product code, so real scans against large dynamic pages benefit, not
just CI.
---
 bbot/core/helpers/diff.py         | 27 ++++++++++++++++++++-------
 bbot/test/test_step_1/test_web.py |  7 ++++---
 2 files changed, 24 insertions(+), 10 deletions(-)

diff --git a/bbot/core/helpers/diff.py b/bbot/core/helpers/diff.py
index 8911edd320..afb0393569 100644
--- a/bbot/core/helpers/diff.py
+++ b/bbot/core/helpers/diff.py
@@ -9,6 +9,24 @@
 log = logging.getLogger("bbot.core.helpers.diff")
 
 
+def _is_flat_str_sequence(content):
+    return isinstance(content, list) and all(isinstance(i, str) for i in content)
+
+
+def _ordered_diff(content_1, content_2, **kwargs):
+    """DeepDiff with ignore_order, skipping pair-matching for flat string lists.
+
+    Pair-matching is O(n*m) nested DeepDiffs over the differing items, which is
+    what a split("\\n") body of a dynamic page looks like. For sequences whose
+    items are all strings there is nothing to match deeper, and equal-index
+    add/remove pairs are folded back into values_changed by DeepDiff itself, so
+    the result is unchanged.
+    """
+    if _is_flat_str_sequence(content_1) and _is_flat_str_sequence(content_2):
+        kwargs["cutoff_intersection_for_pairs"] = 0
+    return DeepDiff(content_1, content_2, ignore_order=True, view="tree", threshold_to_diff_deeper=0, **kwargs)
+
+
 class _BaselineSnapshot:
     """Lightweight stand-in for a blasthttp Response held by HttpCompare.
 
@@ -173,9 +191,7 @@ async def _baseline(self):
                 baseline_1_json = baseline_1.text.split("\n")
                 baseline_2_json = baseline_2.text.split("\n")
 
-            ddiff = DeepDiff(
-                baseline_1_json, baseline_2_json, ignore_order=True, view="tree", threshold_to_diff_deeper=0
-            )
+            ddiff = _ordered_diff(baseline_1_json, baseline_2_json)
             self.ddiff_filters = []
 
             for k in ddiff.keys():
@@ -265,13 +281,10 @@ def compare_body(self, content_1, content_2):
         if sum(differing.values()) > self.max_differing_lines:
             return False
 
-        ddiff = DeepDiff(
+        ddiff = _ordered_diff(
             content_1,
             content_2,
-            ignore_order=True,
-            view="tree",
             exclude_paths=self.ddiff_filters,
-            threshold_to_diff_deeper=0,
         )
 
         if len(ddiff.keys()) == 0:
diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index 0b080b0b0d..b12d55c5a6 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -2,10 +2,11 @@
 import time
 
 from blasthttp import HTTPStatusError
-from deepdiff import DeepDiff
 
 from ..bbot_fixtures import *
 
+from bbot.core.helpers.diff import _ordered_diff
+
 from bbot.test.worker import BBOT_TEST_DIR, HTTPSERVER_HOSTPORT
 
 
@@ -464,7 +465,7 @@ async def test_web_http_compare_filtered_lines_not_counted(blasthttp_mock, bbot_
     baseline_2 = dynamic_b + static
     subject = dynamic_c + static
 
-    ddiff = DeepDiff(baseline_1, baseline_2, ignore_order=True, view="tree", threshold_to_diff_deeper=0)
+    ddiff = _ordered_diff(baseline_1, baseline_2)
     compare_helper.ddiff_filters = [x.path() for k in ddiff.keys() for x in list(ddiff[k])]
     assert len(compare_helper.ddiff_filters) == 600
 
@@ -512,7 +513,7 @@ async def test_web_http_compare_threshold_boundary(blasthttp_mock, bbot_scanner)
     at_threshold_1 = [f"nonce {i} A" for i in range(5)] + static
     at_threshold_2 = [f"nonce {i} B" for i in range(5)] + static
 
-    ddiff = DeepDiff(at_threshold_1, at_threshold_2, ignore_order=True, view="tree", threshold_to_diff_deeper=0)
+    ddiff = _ordered_diff(at_threshold_1, at_threshold_2)
     compare_helper.ddiff_filters = [x.path() for k in ddiff.keys() for x in list(ddiff[k])]
 
     assert compare_helper.compare_body(at_threshold_1, at_threshold_2) is True

From f5a42a9c889b60d5e2754b2022a75b3ecf6afcce Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 07:25:09 -0700
Subject: [PATCH 41/78] perf(ci): cut distro matrix down to the install surface

The distro jobs still ran eight test files on all six containers. Only
three of them probe anything that varies by distro: test_e2e (builds a
venv and runs the real binary as a subprocess), test_depsinstaller
(ansible and the package-manager path), and test_command (subprocess and
sudo handling).

The other five were pure library logic. test_dns, test_cli, test_config,
test_files and test_python_api exercise Python that does not change
between Debian and Fedora, and tests.yml already runs the entire suite on
five Python versions. Running them six more times per PR bought nothing.

Those five were also the entire cost. Per distro they accounted for
roughly 5.5 of the 7-8.5 minutes of test time, with test_dns and test_cli
alone over 300s on every container.

Dropping them also removes the fedora hang from this workflow.
test_dns_helpers reaches the live nameservers list on
raw.githubusercontent.com through resolver_file(), unmocked. When that
fetch stalls it burns two 300s download attempts, which matches the 668s
gap seen on back-to-back fedora runs where four workers blocked and
released together. That download still needs mocking for tests.yml; this
just stops paying for it six times over.

Collection drops from 48 tests to 9.
---
 .github/workflows/distro_tests.yml | 7 +------
 1 file changed, 1 insertion(+), 6 deletions(-)

diff --git a/.github/workflows/distro_tests.yml b/.github/workflows/distro_tests.yml
index 7e56d304cc..1c02631d70 100644
--- a/.github/workflows/distro_tests.yml
+++ b/.github/workflows/distro_tests.yml
@@ -53,13 +53,8 @@ jobs:
           uv sync --group dev
           uv run pytest -n $(uv run python bbot/test/worker_count.py) --dist loadgroup --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO \
             bbot/test/test_step_1/test_e2e.py \
-            bbot/test/test_step_1/test_cli.py \
             bbot/test/test_step_1/test_depsinstaller.py \
-            bbot/test/test_step_1/test_command.py \
-            bbot/test/test_step_1/test_files.py \
-            bbot/test/test_step_1/test_python_api.py \
-            bbot/test/test_step_1/test_config.py \
-            bbot/test/test_step_1/test_dns.py
+            bbot/test/test_step_1/test_command.py
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v7

From d35afff403ca0cb0a7ad772554c8168b1c63613d Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 08:14:54 -0700
Subject: [PATCH 42/78] perf(lightfuzz): stop delay probing once blind sqli is
 confirmed

The time-based blind SQLi loop iterated all 8 DELAY_PROBE_TEMPLATES even
after a template passed both confirmation stages. Every additional template
that also fires costs delay_stage1_reps*delay_low + delay_stage2_reps*delay_high
(3*3s + 3*8s = 33s) and appends a second result for the same injection point.

Several templates differ only by quote context, so a vulnerable parameter
routinely matches more than one. Test_Lightfuzz_sqli_delay_or_rowindependent
matched both the quoted and unquoted "OR SLEEP(n) IS NOT NULL" variants and
emitted two findings whose payloads differed by a single leading quote.

Measured with a standard_probe tracer over the sqli tests:
  or_rowindependent  66.5s probe wall, 60 probes, 2 findings
    -> 33.3s probe wall, 45 probes, 1 finding
  sqli_delay         33.3s, 57 probes -> 33.3s, 42 probes, 1 finding
Non-firing tests are unchanged at 0 findings with identical probe counts,
including the jitter false-positive guard (54 probes, 7.6s, 0 findings).

Full file: 114 passed, 388.82s -> 360.12s. url_manipulation, bypass403 and
test_web: 26 passed.
---
 bbot/modules/lightfuzz/submodules/sqli.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bbot/modules/lightfuzz/submodules/sqli.py b/bbot/modules/lightfuzz/submodules/sqli.py
index 545d54a494..23e944302a 100644
--- a/bbot/modules/lightfuzz/submodules/sqli.py
+++ b/bbot/modules/lightfuzz/submodules/sqli.py
@@ -309,6 +309,8 @@ async def fuzz(self):
                             ),
                         }
                     )
+                    # remaining templates only re-confirm the same injection point
+                    break
                 else:
                     self.verbose(
                         f"Stage 2 rejected {self.event.url}: d_high={d_high:.2f}s, "

From 3aaf24d69774105ef5834c9ef3a1408e616494fc Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 09:00:43 -0700
Subject: [PATCH 43/78] perf(lightfuzz): stop padding oracle probing once the
 verdict is fixed

padding_oracle_execute always sent all 254 probes even when the outcome
was already decided. The return value is a function of differ_count and
block_size only: once differ_count exceeds block_size the result is
pinned to None (possible_first_byte) or False, and no further increment
can move it. Break there instead of finishing the sweep.

Verified exhaustively over every (block_size, possible_first_byte,
break-point, reachable-final-count) combination: 0 mismatches, so the
early exit is provably equivalent to the full sweep.

Measured on the crypto cluster, per-block-size verdicts byte-identical
before and after:
  CBCBitflipDetection_NoPaddingOracle    1020 -> 56 probes, 8.37s -> 2.41s
  envelope_isolation_cbc_bitflip_no_po   1020 -> 56 probes, 7.90s -> 2.43s
  PaddingOracleDetection_Noisy            767 -> 492 probes, 5.35s -> 4.26s
Confirming cases are untouched: they return True at 254 probes, exactly
as before, so detection sensitivity is unchanged.

Also fixed a duplicate-finding defect in padding_oracle: it kept testing
remaining block sizes after appending a result, so one vulnerable
parameter could emit multiple Padding Oracle findings differing only by
block size. cbc_bitflip already returns after its first match; this makes
padding_oracle consistent with it. Affects real scans, not just tests.

Full file: 114 passed, 355.33s -> 333.84s.
---
 bbot/modules/lightfuzz/submodules/crypto.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bbot/modules/lightfuzz/submodules/crypto.py b/bbot/modules/lightfuzz/submodules/crypto.py
index 085589c1d5..291a7ea208 100644
--- a/bbot/modules/lightfuzz/submodules/crypto.py
+++ b/bbot/modules/lightfuzz/submodules/crypto.py
@@ -567,6 +567,9 @@ async def padding_oracle_execute(self, original_data, encoding, block_size, cook
                     if char_diffs <= 5:
                         continue
                 differ_count += 1
+                # verdict is already fixed above this threshold; further probes cannot change it
+                if differ_count > block_size:
+                    break
         self.debug(f"padding_oracle_execute: finished loop. differ_count={differ_count}")
         # A padding oracle vulnerability can produce a small number of different responses.
         # The correct \x01 padding byte always differs, but also, multi-byte padding values (\x02\x02, \x03\x03\x03, etc.) can also produce valid padding if the intermediate state randomly aligns. At most 'block_size' of such values are possible.
@@ -648,6 +651,7 @@ async def padding_oracle(self, probe_value, cookies):
                         "context": context,
                     }
                 )
+                return  # Report first matching block size only
 
     async def error_string_search(self, text_dict, baseline_text):
         """

From 9bd71773a8b329d75accff9be13a375fbaaa003a Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 09:42:18 -0700
Subject: [PATCH 44/78] perf(test): build the e2e venv once instead of once per
 worker

test_e2e.py's bbot_venv fixture is module-scoped, but xdist scopes fixtures
per worker, not per session. With --dist loadgroup the four TestE2E tests
were free to land on four different workers, so the fixture was constructed
once on each of them. Each construction runs python -m venv plus
pip install -e on the local checkout, measured at 40s for the pip step
alone.

Confirmed in CI (py3.13 run 32864709722, job 97857382582): test_install_and_help
and test_scan_dns ran on gw2, test_scan_web and test_clean_shutdown on gw3.
test_scan_web was credited 88.7s while the scan it performs takes 8.25s when
run directly against the same two-page local server, and test_install_and_help
was credited 29.1s for a bbot -h. The difference on both is a venv rebuild.

Pinning the class to one xdist group keeps the module together so the venv is
built once. Proved by counting the fixture's tmp dirs after a local -n 4
--dist loadgroup run: four bbot_e2e_venv0 dirs (popen-gw0 through gw3) before,
one (popen-gw0) after.

This is fixture locality, not sharding. All four tests still run, on the same
worker, with every assertion intact: 4 passed both before and after.
---
 bbot/test/test_step_1/test_e2e.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/bbot/test/test_step_1/test_e2e.py b/bbot/test/test_step_1/test_e2e.py
index ee030b8375..25745f1029 100644
--- a/bbot/test/test_step_1/test_e2e.py
+++ b/bbot/test/test_step_1/test_e2e.py
@@ -106,6 +106,9 @@ def run_bbot(bbot_bin, *args, timeout=180):
     )
 
 
+# bbot_venv is module-scoped but xdist scopes fixtures per worker, so splitting
+# these across workers rebuilds the venv once per worker
+@pytest.mark.xdist_group("e2e_venv")
 class TestE2E:
     def test_install_and_help(self, bbot_venv):
         """bbot installs from local source and -h works."""

From 3aa43a7f8a63fcf3ada9c45e9d36ffb749d5e91f Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 10:05:02 -0700
Subject: [PATCH 45/78] fix(ci): run benchmarks once and fail loudly

The benchmark job checked out the base branch, ran the full suite, then
checked out the head branch and ran it again. On a fork PR the second
checkout could never work: actions/checkout only fetches refs from
origin, so the head branch does not exist locally and git exits with
"pathspec did not match any file(s) known to git".

Every run today burned 14 minutes benchmarking the base branch, crashed
before the head branch, and reported success anyway because the step
carried continue-on-error: true. The PR comment step is gated on the head
repo matching, so nothing surfaced. Twelve consecutive green runs
produced no report.

Drop the branch comparison. The script now benchmarks the checked-out
tree once and reports it. No checkouts, so no git state to clean up or
restore, and no way to fail on a ref that was never fetched.

Make failure visible. run_benchmarks returns False on a non-zero pytest
exit instead of accepting whatever pytest-benchmark managed to write,
main returns an exit code, and continue-on-error is gone.

Verified locally: 31 passed, report written, working tree and branch
untouched. A seeded failure run exits non-zero instead of reporting
success.
---
 .github/workflows/benchmark.yml  |  16 ++---
 bbot/scripts/benchmark_report.py | 118 ++++++++++---------------------
 2 files changed, 40 insertions(+), 94 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 6d36ec88bb..3e4257dd05 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -41,17 +41,11 @@ jobs:
         sudo apt-get install -y libmagic1
 
     # Generate benchmark comparison report using our branch-based script
-    - name: Generate benchmark comparison report
-      env:
-        BASE_REF: ${{ github.base_ref }}
-        HEAD_REF: ${{ github.head_ref }}
+    - name: Generate benchmark report
       run: |
         uv run python bbot/scripts/benchmark_report.py \
-          --base "$BASE_REF" \
-          --current "$HEAD_REF" \
           --output benchmark_report.md \
           --keep-results
-      continue-on-error: true
 
     # Upload benchmark results as artifacts
     - name: Upload benchmark results
@@ -60,8 +54,7 @@ jobs:
         name: benchmark-results
         path: |
           benchmark_report.md
-          base_benchmark_results.json
-          current_benchmark_results.json
+          benchmark_results.json
         retention-days: 30
 
     # Comment on PR with benchmark results.
@@ -144,10 +137,9 @@ jobs:
             console.error('Failed to read benchmark report:', e.message);
             report = `## Performance Benchmark Report
 
-          > **Failed to generate detailed benchmark comparison**
+          > **Failed to generate benchmark report**
           >
-          > The benchmark comparison failed to run. This might be because:
-          > - Benchmark tests don't exist on the base branch yet
+          > This might be because:
           > - Dependencies are missing
           > - Test execution failed
           >
diff --git a/bbot/scripts/benchmark_report.py b/bbot/scripts/benchmark_report.py
index a5b06aec92..3f2290901c 100644
--- a/bbot/scripts/benchmark_report.py
+++ b/bbot/scripts/benchmark_report.py
@@ -6,6 +6,7 @@
 a comparison report showing performance differences between them.
 """
 
+import sys
 import json
 import argparse
 import subprocess
@@ -32,20 +33,6 @@ def get_current_branch() -> str:
     return result.stdout.strip()
 
 
-def checkout_branch(branch: str, repo_path: Path = None):
-    """Checkout a git branch, cleaning up generated and modified files first."""
-    # Reset modified tracked files (e.g. uv.lock changed by `uv sync`)
-    print("Resetting modified tracked files before checkout")
-    run_command(["git", "checkout", "--", "."], cwd=repo_path)
-    # Remove untracked files before checkout. Without this, files generated
-    # by one branch's toolchain (e.g. uv.lock from `uv run` on a Poetry
-    # branch) block checkout to a branch that tracks those same files.
-    print("Cleaning untracked files before checkout")
-    run_command(["git", "clean", "-fd"], cwd=repo_path)
-    print(f"Checking out branch: {branch}")
-    run_command(["git", "checkout", branch], cwd=repo_path)
-
-
 def run_benchmarks(output_file: Path, repo_path: Path = None) -> bool:
     """Run benchmarks and save results to JSON file.
 
@@ -82,9 +69,8 @@ def run_benchmarks(output_file: Path, repo_path: Path = None) -> bool:
 
     if result.returncode != 0:
         print(f"Pytest exited with code {result.returncode}")
+        return False
 
-    # pytest-benchmark writes JSON regardless of test failures;
-    # treat the run as successful if the output file has data
     if output_file.exists() and output_file.stat().st_size > 0:
         return True
 
@@ -379,15 +365,13 @@ def generate_report(current_data: Dict, base_data: Dict, current_branch: str, ba
     if not base_data:
         report = f"""## 🚀 Performance Benchmark Report
 
-> ℹ️ **No baseline benchmark data available**
-> 
-> Showing current results for **{current_branch}** only.
+> Results for **{current_branch}**.
 
 """
         current_benchmarks = current_data.get("benchmarks", [])
         if current_benchmarks:
             report += f"""<details>
-<summary>📊 Current Results ({current_branch}) - Click to expand</summary>
+<summary>📊 Results ({current_branch}) - Click to expand</summary>
 
 {generate_benchmark_table(current_benchmarks, "Results")}
 </details>"""
@@ -416,79 +400,49 @@ def generate_report(current_data: Dict, base_data: Dict, current_branch: str, ba
 
 
 def main():
-    parser = argparse.ArgumentParser(description="Compare benchmark performance between git branches")
-    parser.add_argument("--base", required=True, help="Base branch name (e.g., 'main', 'dev')")
-    parser.add_argument("--current", required=True, help="Current branch name (e.g., 'feature-branch', 'HEAD')")
+    parser = argparse.ArgumentParser(description="Run BBOT performance benchmarks and report the results")
     parser.add_argument("--output", type=Path, help="Output markdown file (default: stdout)")
     parser.add_argument("--keep-results", action="store_true", help="Keep intermediate JSON files")
 
     args = parser.parse_args()
 
-    # Get current working directory
     repo_path = Path.cwd()
 
-    # Save original branch to restore later
     try:
-        original_branch = get_current_branch()
-        print(f"Current branch: {original_branch}")
+        branch = get_current_branch()
     except subprocess.CalledProcessError:
-        print("Warning: Could not determine current branch")
-        original_branch = None
+        branch = "HEAD"
 
-    # Create temporary files for benchmark results
     with tempfile.TemporaryDirectory() as temp_dir:
-        temp_path = Path(temp_dir)
-        base_results_file = temp_path / "base_results.json"
-        current_results_file = temp_path / "current_results.json"
-
-        base_data = {}
-        current_data = {}
-
-        try:
-            # Run benchmarks on base branch
-            print(f"\n=== Running benchmarks on base branch: {args.base} ===")
-            checkout_branch(args.base, repo_path)
-            if run_benchmarks(base_results_file, repo_path):
-                base_data = load_benchmark_data(base_results_file)
-
-            # Run benchmarks on current branch
-            print(f"\n=== Running benchmarks on current branch: {args.current} ===")
-            checkout_branch(args.current, repo_path)
-            if run_benchmarks(current_results_file, repo_path):
-                current_data = load_benchmark_data(current_results_file)
-
-            # Generate report
-            print("\n=== Generating comparison report ===")
-            report = generate_report(current_data, base_data, args.current, args.base)
-
-            # Output report
-            if args.output:
-                with open(args.output, "w") as f:
-                    f.write(report)
-                print(f"Report written to {args.output}")
-            else:
-                print("\n" + "=" * 80)
-                print(report)
-
-            # Keep results if requested
-            if args.keep_results:
-                if base_data:
-                    with open("base_benchmark_results.json", "w") as f:
-                        json.dump(base_data, f, indent=2)
-                if current_data:
-                    with open("current_benchmark_results.json", "w") as f:
-                        json.dump(current_data, f, indent=2)
-                print("Benchmark result files saved.")
-
-        finally:
-            # Restore original branch
-            if original_branch:
-                print(f"\nRestoring original branch: {original_branch}")
-                try:
-                    checkout_branch(original_branch, repo_path)
-                except subprocess.CalledProcessError:
-                    print(f"Warning: Could not restore original branch {original_branch}")
+        results_file = Path(temp_dir) / "results.json"
+
+        print(f"\n=== Running benchmarks on {branch} ===")
+        if not run_benchmarks(results_file, repo_path):
+            print("Benchmarks produced no data")
+            return 1
+
+        data = load_benchmark_data(results_file)
+        if not data:
+            print(f"No benchmark data in {results_file}")
+            return 1
+
+        report = generate_report(data, {}, branch, None)
+
+        if args.output:
+            with open(args.output, "w") as f:
+                f.write(report)
+            print(f"Report written to {args.output}")
+        else:
+            print("\n" + "=" * 80)
+            print(report)
+
+        if args.keep_results:
+            with open("benchmark_results.json", "w") as f:
+                json.dump(data, f, indent=2)
+            print("Benchmark result files saved.")
+
+    return 0
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

From 56fb7252b4d44c2205fbdd1c08058a7ea98989e3 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 10:45:09 -0700
Subject: [PATCH 46/78] perf(wayback): consume archive cache entries as they
 are fetched

finish() built its fetch list from _archive_cache but never removed the
entries it processed. The scan's finish phase is re-entrant by design:
base.py re-invokes finish() on every FINISHED event, and finish() itself
emits HTTP_RESPONSE events, so it re-triggers itself. Every re-entry
refetched the full archive set from scratch.

The second pass is pure waste. Its URLs were already emitted, so the
bloom filter rejects them as duplicates and nothing new reaches the
scan. But the fetch still happens first, and against 429 or error
responses it pays the whole retry budget again: three per-request
retries plus a 30s batch retry backoff, for URLs already fetched
successfully.

TestWaybackArchive429Retry made this visible because it exercises the
retry path. The test drops the 429 delay to 0.01s, but the batch retry
backoff is a hardcoded 30 * retry_num, so the redundant pass burned 30s
that no test knob could reach.

Delete each entry as it is consumed. Unpaired entries (a string, not yet
matched to a parent event) are left in place, since a later parent event
can still pair them.

Verified: finish() archive passes across the file drop from 12 to 6, one
per archive-enabled test. The "Archive loading complete" lines lose
exactly the failing duplicates (5x "0/1" and 1x "0/2") and keep every
successful pass. 429 handling, retry, and emission each still run once.
TestWaybackArchive429Retry 41.26s -> 2.93s, whole file 78.63s -> 38.45s.
wayback + unarchive + excavate: 73 passed.
---
 bbot/modules/wayback.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/bbot/modules/wayback.py b/bbot/modules/wayback.py
index e58f6d9a37..eef2da22a7 100644
--- a/bbot/modules/wayback.py
+++ b/bbot/modules/wayback.py
@@ -562,12 +562,16 @@ async def finish(self):
         skip_extensions.update(e.lower() for e in self.scan.config.get("url_extension_special", []))
 
         # build URL list and mapping back to metadata
+        # entries are consumed here: emitting events re-triggers the finish phase, and
+        # anything left in the cache would be fetched again on every re-entry
         url_metadata = {}
         for cleaned_url, value in list(self._archive_cache.items()):
             if not isinstance(value, tuple):
+                # unpaired entries stay put; a later parent event can still pair them
                 self.debug(f"Skipping unpaired archive entry: {cleaned_url}")
                 continue
             raw_url, parent_event = value
+            del self._archive_cache[cleaned_url]
             ext = get_file_extension(cleaned_url)
             if ext and ext in skip_extensions:
                 self.debug(f"Skipping archive fetch for {raw_url} (extension: .{ext})")

From 6cc2f0831f9f468748de95d109d689657f529f2b Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 11:54:26 -0700
Subject: [PATCH 47/78] fix(scanner): hard-fail modules whose setup() raises

A module whose setup() raised was reported as "Setup succeeded" and left in
the scan in a permanently errored state. This hung CI: the py3.11 job at
3aaf24d69 ran 37 minutes and was only stopped by auto-cancel, and py3.13 at
9bd71773a sat over two hours. Both stalled on TestNucleiRetries.

_setup() initializes status=False, then overwrites it per function in the
funcs loop. When setup() raises, the except branch sets the message but never
resets status, so it keeps the value left by the preceding setup_deps() call,
which is True. setup_modules() then reads status is True, logs "Setup
succeeded for excavate (dictionary changed size during iteration)", and skips
the removal at scanner.py:635. The errored module stays registered, and since
set_error_state() makes it refuse new events, the scan never reaches
modules_finished. The debug artifact shows excavate(3:0:0) held 3 events with
"Modules errored: 1 (excavate)" repeating every 15s until the runner died.

Nothing interrupts this: the suite runs with timeout_func_only=true, so the
1200s timeout covers only the test function, not fixture setup where the scan
is driven from. A fixture hang runs forever instead of failing.

Set status = False in the except branch so a raising setup() is treated as
the hard failure it is.

The trigger was a second defect in excavate.setup(), fixed here too. It
iterated self.scan.modules directly, twice. setup_modules() runs every
module's _setup() concurrently under as_completed and pops failed modules
from that same dict, so excavate could be walking it as an entry was
removed, raising "dictionary changed size during iteration". Snapshot the
values once up front.

Verified:
- Reproduced the status bug in isolation: raising setup() returned
  (True, "dictionary changed size during iteration"), matching the CI log
  line exactly. Now returns False with errored=True.
- Negative control on the stashed pre-fix tree returns True, confirming the
  one-line change is load-bearing.
- module.name is provably identical to the dict key it replaces: 15 modules
  in a real scan, zero mismatches, identical WEB_PARAMETER list both ways.
- test_modules_basic 7 passed, test_module_excavate 52 passed, paramminer
  getparams/headers/cookies 17 passed, test_scan 13 passed.
- ruff check and ruff format --check clean on both files.
---
 bbot/modules/base.py              |  3 +++
 bbot/modules/internal/excavate.py | 12 ++++++------
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/bbot/modules/base.py b/bbot/modules/base.py
index 4b911f3d89..33b256007b 100644
--- a/bbot/modules/base.py
+++ b/bbot/modules/base.py
@@ -720,6 +720,9 @@ async def _setup(self, deps_only=False):
                     break
             self.debug(f"Finished setting up module {self.name}")
         except Exception as e:
+            # a raising setup() must hard-fail, otherwise setup_modules() reads the
+            # last successful status and leaves an errored module in the scan
+            status = False
             self.set_error_state(f"Unexpected error during module setup: {e}", critical=True)
             msg = f"{e}"
             self.trace()
diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index 0d1651c375..cdc5c4ac5e 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -1372,11 +1372,11 @@ async def setup(self):
         self.yara_preprocess_dict = {}
         self.custom_yara_imports = []
 
-        modules_WEB_PARAMETER = [
-            module_name
-            for module_name, module in self.scan.modules.items()
-            if "WEB_PARAMETER" in module.watched_events
-        ]
+        # snapshot: setup_modules() pops failed modules from scan.modules while
+        # these setups run concurrently, so iterating the live dict can raise
+        scan_modules = list(self.scan.modules.values())
+
+        modules_WEB_PARAMETER = [module.name for module in scan_modules if "WEB_PARAMETER" in module.watched_events]
 
         self.parameter_extraction = bool(modules_WEB_PARAMETER)
         self.speculate_params = self.config.get("speculate_params", False)
@@ -1385,7 +1385,7 @@ async def setup(self):
         # at each YARA form-opening match. Caps worst-case Python re work per form.
         self.max_form_bytes = int(self.config.get("max_form_bytes", 262144))
 
-        for module in self.scan.modules.values():
+        for module in scan_modules:
             if not str(module).startswith("_"):
                 ExcavateRules = find_subclasses(module, ExcavateRule)
                 for e in ExcavateRules:

From a5005c43fc639c5d1403ddfabfef81f3caccf570 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 12:31:57 -0700
Subject: [PATCH 48/78] perf(deps): let retry_failed use the lock-free precheck

The install fast path added in b9bdb4e25 was dead for the entire test
suite. bbot/test/test.conf sets deps.behavior: retry_failed, and
_all_deps_satisfied() bailed out immediately for that mode, so every
scan fell through to the exclusive flock on install.lock. Under xdist
that serializes all four workers behind whichever one is installing.

test_cli_args runs --install-all-deps and holds the lock for ~150s.
Any worker calling _prep() in that window blocked for the remainder,
and the resulting stall was charged to whichever innocent test happened
to be running. That is why the number two slot moved between jobs of
the same run: test_dns_graph_structure 107.9s on 3.12,
test_event_discovery_context 98.8s on 3.10 and 100.4s on 3.11,
test_event_web_spider_distance 123.0s on 3.14. All four overlap
test_cli_args at 97 to 100 percent, and all run in about 1s alone.

The GitHub log timestamps make this look like idle workers, but the
pytest_debug.log artifact shows the victims logging "Executing
scan._prep()" and then nothing until the CLI install writes
"Successfully installed dependencies for 142 modules", after which they
resume within the same second.

retry_failed differs from the default only in that it revisits modules
recorded as False. The precheck loop already declines on any status
that is not True, so it cannot skip a retry. force_install still
declines unconditionally.

Verified: with the lock held, _prep() under retry_failed goes 15.56s to
0.61s. A module marked False still declines the fast path and gets
retried, force_install still declines, and the fast path agrees with
_install() on all 149 modules, 0 disagreements.

test_depsinstaller fails locally on a clean tree (ansible/sudo), and is
unaffected by this change.
---
 bbot/core/helpers/depsinstaller/installer.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index 259c156bcb..0d83702b37 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -180,7 +180,10 @@ def _all_deps_satisfied(self, modules):
         Mirrors the accounting in _install() so the fast path and the locked path
         agree on which modules count as succeeded.
         """
-        if self.deps_behavior in ("force_install", "retry_failed"):
+        # retry_failed only revisits modules recorded as failed; the loop below already
+        # declines on any status that is not True, so it is safe on this path.
+        # force_install reinstalls regardless of status, so it can never skip the lock.
+        if self.deps_behavior == "force_install":
             return None
         if not self._core_deps_cached():
             return None

From 1c411ac94d47da9912c1f6bd264f64a3ae1e6253 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 12:52:57 -0700
Subject: [PATCH 49/78] perf(webbrute): memoize the fuzz baseline per shape

`baseline_fuzz()` built a fresh `HttpCompare` and probed the target on
every call, including for a shape it had already characterized. A
baseline describes the response to a nonexistent path of a given shape,
so it is a pure function of (url, prefix, suffix, ext). Re-probing an
identical shape costs two HTTP requests plus the 0.5s inter-sample sleep
in `HttpCompare._baseline()` and returns the same filter.

webbrute_shortnames fans out hard on this. Each URL_HINT triggers a run
for the plain hint, then again for the detected delimiter, then again
for the detected subword, then once more per common prefix in `finish()`
-- each against the same host with the same extension set. Profiling
test_module_webbrute_shortnames showed 42 `baseline_fuzz()` calls
collapsing to only 16 distinct shapes, so 50 baselines were established
where 24 requests' worth of work was needed. Baseline time dominated the
test: 28.3s of a 32s run.

Shapes are now memoized per module instance behind a per-key lock, so
concurrent handlers racing on a cold shape collapse onto one probe
instead of each running their own. Only successful baselines are cached;
an abort verdict (CONNECTIVITY_ISSUES, WAF_BLOCK_PAGE, RECEIVED_429) is
returned but never stored, so a transient failure cannot poison later
events, and host-level aborts still short-circuit the remaining
extensions exactly as before. The cache is an LRU bounded at 1000
entries because each entry pins an HttpCompare and its baseline
snapshot, and a real scan spans many hosts.

test_module_webbrute_shortnames: 33s -> 27s, baselines 50 -> 24.

Verified:
- Cache invariants asserted directly: identical shape reuses the same
  HttpCompare object, four distinct shapes take four probes, 10
  concurrent callers on a cold shape yield exactly 1 probe, LRU holds at
  the bound across 50 hosts, and both abort paths stay uncached while
  still propagating host_abort to every remaining ext.
- NEGATIVE CONTROL on the stashed pre-fix tree: the reuse assertion
  fails with 2 probes for one shape. Load-bearing.
- webbrute 11, webbrute_shortnames 1, iis_shortnames 2, test_web,
  paramminer getparams/headers: 50 passed, 0 failed. Every
  false-positive defense test (canary, mid-scan drift, hit cap, WAF,
  redirect, wildcard skip) still passes.
- ruff check + ruff format --check clean.
---
 bbot/modules/webbrute.py | 80 ++++++++++++++++++++++++++++------------
 1 file changed, 57 insertions(+), 23 deletions(-)

diff --git a/bbot/modules/webbrute.py b/bbot/modules/webbrute.py
index 441848f465..55454f2022 100644
--- a/bbot/modules/webbrute.py
+++ b/bbot/modules/webbrute.py
@@ -1,6 +1,8 @@
 import random
 import string
+import asyncio
 from typing import Union
+from collections import OrderedDict
 
 import blasthttp
 
@@ -45,6 +47,15 @@ class Config(BaseModuleConfig):
     in_scope_only = True
     _module_threads = 4
 
+    # bounds the per-shape baseline cache; each entry pins an HttpCompare and its baseline snapshot
+    _baseline_cache_size = 1000
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        # keyed by (url, prefix, suffix, ext); subclasses that override setup() inherit this
+        self._baseline_cache = OrderedDict()
+        self._baseline_locks = OrderedDict()
+
     async def setup_deps(self):
         self.wordlist = await self.helpers.wordlist(self.config.get("wordlist"))
         return True
@@ -128,16 +139,28 @@ def _build_batch_headers(self):
     # Host-level abort reasons that apply to ALL extensions, not just the one that triggered them.
     HOST_ABORT_REASONS = {"WAF_BLOCK_PAGE", "CONNECTIVITY_ISSUES", "BASELINE_CHANGED_CODES", "RECEIVED_429"}
 
-    async def baseline_fuzz(self, url, exts=None, prefix="", suffix=""):
-        if exts is None:
-            exts = [""]
-        filters = {}
-        host_abort = None
-
-        for ext in exts:
-            if host_abort is not None:
-                filters[ext] = host_abort
-                continue
+    async def _baseline_for(self, url, prefix, suffix, ext):
+        """Establish (or reuse) the baseline for one fuzz shape.
+
+        A baseline characterizes the response to a nonexistent path of a given
+        shape, so it depends only on (url, prefix, suffix, ext). Re-probing the
+        same shape costs two requests plus the 0.5s inter-sample sleep and
+        yields the same filter, so shapes are memoized per module instance and
+        raced callers are collapsed onto one probe.
+        """
+        key = (url, prefix, suffix, ext)
+        lock = self._baseline_locks.get(key)
+        if lock is None:
+            lock = self._baseline_locks[key] = asyncio.Lock()
+            while len(self._baseline_locks) > self._baseline_cache_size:
+                self._baseline_locks.popitem(last=False)
+
+        async with lock:
+            cached = self._baseline_cache.get(key)
+            if cached is not None:
+                self._baseline_cache.move_to_end(key)
+                self.debug(f"reusing baseline for URL [{url}] with ext [{ext}]")
+                return cached, None
 
             self.debug(f"running baseline for URL [{url}] with ext [{ext}]")
 
@@ -157,33 +180,44 @@ async def baseline_fuzz(self, url, exts=None, prefix="", suffix=""):
                 await compare._baseline()
             except HttpCompareError as e:
                 self.warning(f"Could not establish baseline for URL [{url}] ext [{ext}]: {e}")
-                abort = {"abort": True, "reason": "CONNECTIVITY_ISSUES"}
-                filters[ext] = abort
-                host_abort = abort
-                continue
+                return {"abort": True, "reason": "CONNECTIVITY_ISSUES"}, "CONNECTIVITY_ISSUES"
 
             baseline_status = compare.baseline.status_code
 
             if await self.helpers.yara.match(self.waf_yara_rules, compare.baseline.content):
                 self.warning(f"Baseline for URL [{url}] ext [{ext}] returned WAF block page, aborting.")
-                abort = {"abort": True, "reason": "WAF_BLOCK_PAGE"}
-                filters[ext] = abort
-                host_abort = abort
-                continue
+                return {"abort": True, "reason": "WAF_BLOCK_PAGE"}, "WAF_BLOCK_PAGE"
 
             if baseline_status == 429:
                 self.warning(
                     f"Received 429 (Too Many Requests) for URL [{url}]. A WAF or rate limiter is blocking requests, aborting."
                 )
-                abort = {"abort": True, "reason": "RECEIVED_429"}
-                filters[ext] = abort
-                host_abort = abort
-                continue
+                return {"abort": True, "reason": "RECEIVED_429"}, "RECEIVED_429"
 
             if baseline_status == 403:
                 self.warning("All baseline requests received 403. A WAF may be actively blocking traffic.")
 
-            filters[ext] = {"compare": compare}
+            result = {"compare": compare}
+            self._baseline_cache[key] = result
+            while len(self._baseline_cache) > self._baseline_cache_size:
+                self._baseline_cache.popitem(last=False)
+            return result, None
+
+    async def baseline_fuzz(self, url, exts=None, prefix="", suffix=""):
+        if exts is None:
+            exts = [""]
+        filters = {}
+        host_abort = None
+
+        for ext in exts:
+            if host_abort is not None:
+                filters[ext] = host_abort
+                continue
+
+            result, abort_reason = await self._baseline_for(url, prefix, suffix, ext)
+            filters[ext] = result
+            if abort_reason is not None:
+                host_abort = result
 
         return filters
 

From 24eb5e2696443c14371d19bfcc09a2fa175a4a24 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 25 Aug 2026 13:37:59 -0700
Subject: [PATCH 50/78] perf(trufflehog): stop the binary re-execing itself on
 every invocation

Every trufflehog invocation forked a second copy of itself and paid the
Go package init twice. The re-exec comes from the overseer self-update
library: main() hands control to overseer.RunErr(), which starts the real
program as a child process with OVERSEER_IS_SLAVE=1 in its environment.
Both parent and child run the full init chain, and that chain is not
cheap: 1616 packages, dominated by go-re2 at 488ms compiling the detector
regex set.

Measured on the pinned 3.97.0 binary: a stdin scan of a single line takes
4.6s wall, of which the scan itself is 1.8ms per trufflehog's own
"finished scanning" log. GODEBUG=inittrace=1 shows 3232 init lines for a
real run against 1615 for an invalid flag, exactly 1616 packages
initialized twice, and ps shows the second pid appearing 2.5s in.

trufflehog exposes --local-dev for precisely this, it skips overseer and
runs the program directly. --no-update already told it not to fetch
updates, so the overseer wrapper was pure overhead in every bbot code path.

bbot spawns one process per event, so the cost is paid per CODE_REPOSITORY,
FILESYSTEM, HTTP_RESPONSE and RAW_TEXT event, not once per scan.

Verified:
- Output equivalence across every source mode bbot uses (stdin, filesystem,
  git, plus aws/slack/http secret fixtures): DetectorName, DecoderName, Raw,
  RawV2, Verified and SourceMetadata identical, same returncode, same
  result counts.
- The one field that differs under git, repository_local_path, embeds the
  pid of the scanning process and already differs between two consecutive
  plain runs. Nothing in bbot reads it.
- Binary is not modified and no update is attempted, checked with and
  without --no-update; mtime and size unchanged, zero update-related
  stderr.
- Per invocation 4.6s -> 2.4s, a 1.95x speedup measured over 6 source modes.
- In the test suite, subprocess time 103.2s -> 59.7s over 20 invocations.
- NEGATIVE CONTROL, same file with the change stashed: 63.0s vs 42.7s,
  5 passed both ways. This is the proof the change is load-bearing.
- test_module_trufflehog, test_module_badsecrets, test_module_web_report,
  test_module_github_codesearch, test_modules_basic all pass.
- ruff check + ruff format --check clean.

This is product code, so real scans get the same 2x on every trufflehog
call, not just CI.
---
 bbot/modules/trufflehog.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/bbot/modules/trufflehog.py b/bbot/modules/trufflehog.py
index ee78e2ce41..13824264b4 100644
--- a/bbot/modules/trufflehog.py
+++ b/bbot/modules/trufflehog.py
@@ -149,6 +149,8 @@ async def execute_trufflehog(self, module, path=None, stdin_data=None):
         command = [
             "trufflehog",
             "--json",
+            # without this, overseer re-execs the binary and pays Go package init twice
+            "--local-dev",
             "--no-update",
         ]
         if self.verified:

From 28216d86afd42bb4857cfc257c724403116de5c5 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Wed, 26 Aug 2026 17:45:00 -0700
Subject: [PATCH 51/78] perf(test): stop test_async_helpers idling for 25
 seconds

as_completed defaults to max_concurrent=20, so the wall time of this test
was exactly sum(sleeps)/20. With 1000 sleeps drawn from random.random()
that sum is ~496s, giving a measured 25.3s of pure idling. Nothing was
being exercised during it: the coroutines only slept.

Divide the sleeps by 100. The scheduling behavior under test is unchanged,
only the constant shrinks. Verified across divisors 1, 20, 50, 100 and 200
that completion stays out of submission order and all 1000 results still
come back distinct, so scaling does not collapse the test into a trivially
ordered drain.

While here, cover the branches this test was walking past. It only ever
checked the happy path at the default limit, leaving three untested:

- max_concurrent is now asserted to be saturated, not merely respected.
  Peak in-flight count is exactly the limit, confirmed stable at 1, 5, 20
  and 50 over 25 runs each.
- max_concurrent=None (the unlimited path) was never entered.
- a raising coroutine is yielded rather than swallowed, and does not abort
  the remaining tasks. as_completed has explicit exception handling that
  nothing exercised.

Also assert completion order differs from submission order, which is the
actual contract and was previously unchecked because results went straight
into a set.

Both new assertion groups verified load-bearing by mutation: forcing the
limit to 3 fails on peak == limit, and swallowing raised tasks fails on
len(yielded) == 4.

test_async_helpers 25.33s -> 1.82s. Full file 20 passed, ~45s -> 21.4s.
---
 bbot/test/test_step_1/test_helpers.py | 68 ++++++++++++++++++++++++++-
 1 file changed, 66 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_1/test_helpers.py b/bbot/test/test_step_1/test_helpers.py
index d2a8505ba0..ef46648825 100644
--- a/bbot/test/test_step_1/test_helpers.py
+++ b/bbot/test/test_step_1/test_helpers.py
@@ -815,17 +815,81 @@ async def test_async_helpers():
     import random
     from bbot.core.helpers.misc import as_completed
 
+    # sleeps are scaled down; wall time here is sum(sleeps)/max_concurrent,
+    # so full-second sleeps buy nothing but 25s of idling
     async def do_stuff(r):
-        await asyncio.sleep(r)
+        await asyncio.sleep(r / 100)
         return r
 
     random_ints = [random.random() for _ in range(1000)]
     tasks = [do_stuff(r) for r in random_ints]
     results = set()
+    completion_order = []
     async for t in as_completed(tasks):
-        results.add(await t)
+        r = await t
+        results.add(r)
+        completion_order.append(r)
     assert len(results) == 1000
     assert sorted(random_ints) == sorted(results)
+    # results must arrive as they finish, not in submission order
+    assert completion_order != random_ints
+
+    # max_concurrent is an upper bound that actually gets saturated
+    for limit in (1, 5, 20, 50):
+        live = 0
+        peak = 0
+
+        async def probe(r):
+            nonlocal live, peak
+            live += 1
+            peak = max(peak, live)
+            try:
+                await asyncio.sleep(r / 100)
+                return r
+            finally:
+                live -= 1
+
+        vals = [random.random() for _ in range(200)]
+        completed = 0
+        async for t in as_completed([probe(r) for r in vals], max_concurrent=limit):
+            await t
+            completed += 1
+        assert completed == 200
+        assert peak == limit
+
+    # unlimited concurrency schedules everything at once
+    live = 0
+    peak = 0
+
+    async def unlimited(i):
+        nonlocal live, peak
+        live += 1
+        peak = max(peak, live)
+        try:
+            await asyncio.sleep(0)
+            return i
+        finally:
+            live -= 1
+
+    completed = 0
+    async for t in as_completed([unlimited(i) for i in range(200)], max_concurrent=None):
+        await t
+        completed += 1
+    assert completed == 200
+    assert peak == 200
+
+    # a raising coroutine is still yielded, and does not abort the rest
+    async def boom():
+        raise ValueError("boom")
+
+    async def fine(i):
+        await asyncio.sleep(0)
+        return i
+
+    yielded = [t async for t in as_completed([boom(), fine(1), boom(), fine(2)])]
+    assert len(yielded) == 4
+    assert sum(1 for t in yielded if isinstance(t.exception(), ValueError)) == 2
+    assert sorted(t.result() for t in yielded if t.exception() is None) == [1, 2]
 
 
 def test_portparse(helpers):

From 2f403286278474cf91d5d8b53e220494ca1f74f7 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Sun, 30 Aug 2026 22:55:58 -0700
Subject: [PATCH 52/78] fix(test): stop websocket test wedging the py3.13 job
 for 6 hours

The py3.13 job on PR #3404 (job 98941202149) ran 6h0m18s and was killed by
the workflow limit. The scan itself finished in under a second: the debug
artifact shows "Finished websocket module test" and "No unfinished tasks
detected", then "Cancelling 2 orphaned tasks after websocket" and nothing
further. Three of four xdist workers went idle and the run never completed.

The test served on a task and cancelled it in check(). Server.close() does
not close synchronously, it spawns an internal _close() task which is what
eventually resolves closed_waiter. The fixture teardown in base.py cancels
every remaining task and gathers them, so it cancelled that internal task
mid-flight. wait_closed() awaits closed_waiter under asyncio.shield, so once
its resolver is cancelled the shield swallows the cancellation and the await
can never complete. The gather then blocks forever.

Reproduced deterministically outside pytest, 5/5 runs wedged with the server
task cancelled while a connection was open, matching the 2 orphaned tasks the
CI artifact reports (Server._close and the server coroutine).

Fixed by awaiting serve() directly instead of wrapping it in a task, and
shutting the server down through close() plus wait_closed(). That drains the
internal task before teardown runs, so no orphan is left to cancel. Verified
zero orphaned tasks remain after the test, where previously there were two.

Shutdown runs in _execute_scan rather than check() because the fixture and
the test body run on different event loops, and awaiting the server from
check() raises "attached to a different loop". All assertions unchanged.
---
 .../module_tests/test_module_websocket.py      | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_websocket.py b/bbot/test/test_step_2/module_tests/test_module_websocket.py
index 26cd28ecd8..c4eedb7d03 100644
--- a/bbot/test/test_step_2/module_tests/test_module_websocket.py
+++ b/bbot/test/test_step_2/module_tests/test_module_websocket.py
@@ -1,5 +1,4 @@
 import json
-import asyncio
 import logging
 from websockets.asyncio.server import serve
 
@@ -17,20 +16,21 @@ async def websocket_handler(websocket):
         results["events"].append(message)
 
 
-# Define a coroutine for the server
-async def server_coroutine():
-    async with serve(websocket_handler, "127.0.0.1", WEBSOCKET_PORT) as server:
-        await server.serve_forever()
-
-
 class TestWebsocket(ModuleTestBase):
     config_overrides = {"modules": {"websocket": {"url": f"ws://127.0.0.1:{WEBSOCKET_PORT}/testing"}}}
 
     async def setup_before_prep(self, module_test):
-        self.server_task = asyncio.create_task(server_coroutine())
+        self.server = await serve(websocket_handler, "127.0.0.1", WEBSOCKET_PORT)
+
+    async def _execute_scan(self, module_test):
+        # shut down on the fixture's loop, which is the one that created the server
+        try:
+            await super()._execute_scan(module_test)
+        finally:
+            self.server.close()
+            await self.server.wait_closed()
 
     def check(self, module_test, events):
         assert results["path"] == "/testing"
         decoded_events = [json.loads(e) for e in results["events"]]
         assert any(e["type"] == "SCAN" for e in decoded_events)
-        self.server_task.cancel()

From 52e41fa65199769ab5c1bb4eebd5c165add6cf52 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Sun, 30 Aug 2026 23:20:40 -0700
Subject: [PATCH 53/78] fix(test): bound the orphan-task cancel in module
 teardown

The websocket hang (2f4032862) was one instance of a general defect. Fixture
teardown cancels every leftover task and then does an unbounded
`asyncio.gather` on them. A task that absorbs its cancellation, an await
shielded from it whose resolver was itself cancelled first, never completes,
so that gather blocks forever.

Nothing caps it. The workflow passes `--timeout 1200` together with
`-o timeout_func_only=true`, and per pytest-timeout that setting evaluates the
timeout against the test function body only, ignoring fixture time. Teardown
is therefore uncovered, which is how a 20 minute cap silently became the
6 hour job limit on job 98941202149.

Wait with a timeout instead, and warn with the surviving tasks when they
outlast it. Abandoning a stuck task is strictly better than wedging the
worker: the process is torn down at session end regardless.

Verified with a module test that leaves a deliberately wedged orphan. On the
current tree it passes in 15.7s; with this change stashed the same test
returns no result at all and is killed at 240s despite `--timeout 150`,
confirming both the hang and that the func-only timeout cannot catch it.
No assertions changed, nothing split. Regression: 57 passed under `-n 2`.
---
 bbot/test/test_step_2/module_tests/base.py | 13 +++++++++++--
 bbot/test/worker.py                        |  4 ++++
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/base.py b/bbot/test/test_step_2/module_tests/base.py
index b2b27dd49e..c0da1bd074 100644
--- a/bbot/test/test_step_2/module_tests/base.py
+++ b/bbot/test/test_step_2/module_tests/base.py
@@ -9,7 +9,7 @@
 from bbot.scanner import Scanner
 from bbot.core.config.merge import deep_merge
 from bbot.core.helpers.misc import rand_string
-from bbot.test.worker import CONTAINER_READY_TIMEOUT, _OOM_HINT
+from bbot.test.worker import CONTAINER_READY_TIMEOUT, ORPHAN_CANCEL_TIMEOUT, _OOM_HINT
 
 log = logging.getLogger("bbot.test.modules")
 
@@ -129,7 +129,16 @@ async def module_test(
             self.log.debug(f"Cancelling {len(tasks)} orphaned tasks after {self.name}")
             for t in tasks:
                 t.cancel()
-            await asyncio.gather(*tasks, return_exceptions=True)
+            # Bounded: a task that absorbs its cancellation (an await shielded from
+            # it, whose resolver was itself cancelled) never completes, and an
+            # unbounded gather here then hangs the whole worker until the job
+            # limit. pytest-timeout cannot fire in fixture teardown.
+            _, pending = await asyncio.wait(tasks, timeout=ORPHAN_CANCEL_TIMEOUT)
+            if pending:
+                self.log.warning(
+                    f"{len(pending)} orphaned tasks did not exit within {ORPHAN_CANCEL_TIMEOUT}s "
+                    f"after {self.name}, abandoning them: {pending}"
+                )
 
     async def _execute_scan(self, module_test):
         """Execute the scan and collect events. Can be overridden by benchmark classes."""
diff --git a/bbot/test/worker.py b/bbot/test/worker.py
index 19a796fcfc..5368b95c38 100644
--- a/bbot/test/worker.py
+++ b/bbot/test/worker.py
@@ -90,6 +90,10 @@ def worker_dir(base=BASE_BBOT_TEST_DIR):
 
 _OOM_HINT = "A container that exits with code 137 was OOM-killed; it competes with the other test workers for memory."
 
+# A cancelled task that never completes would otherwise block fixture teardown
+# forever, and pytest-timeout does not cover teardown under timeout_func_only.
+ORPHAN_CANCEL_TIMEOUT = int(os.environ.get("BBOT_TEST_ORPHAN_TIMEOUT", "60"))
+
 
 async def wait_for_container(name, connect, timeout=CONTAINER_READY_TIMEOUT):
     """Retry ``connect`` until it succeeds, or raise once ``timeout`` elapses.

From 3c1125d8d3f9328d789d0a2f9fd6218927a53632 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Sun, 30 Aug 2026 23:37:01 -0700
Subject: [PATCH 54/78] revert(ci): restore benchmark.yml to unblock the push

The benchmark.yml change from 3aa43a7f8 cannot be pushed: the available
OAuth token lacks the `workflow` scope, so the remote rejects any push
whose net diff touches a workflow file. That single mid-stack commit was
gating 11 unpushed commits, two of which cure a 6 hour py3.13 hang.

GitHub evaluates the net diff of the push rather than each commit, so
restoring the file forward puts the workflow back to its origin state and
lets the rest of the stack land. No rebase, no history rewrite.

Reapply once the token has `workflow` scope:
  gh auth refresh -h github.com -s workflow
  git revert 37161606a
---
 .github/workflows/benchmark.yml | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml
index 3e4257dd05..6d36ec88bb 100644
--- a/.github/workflows/benchmark.yml
+++ b/.github/workflows/benchmark.yml
@@ -41,11 +41,17 @@ jobs:
         sudo apt-get install -y libmagic1
 
     # Generate benchmark comparison report using our branch-based script
-    - name: Generate benchmark report
+    - name: Generate benchmark comparison report
+      env:
+        BASE_REF: ${{ github.base_ref }}
+        HEAD_REF: ${{ github.head_ref }}
       run: |
         uv run python bbot/scripts/benchmark_report.py \
+          --base "$BASE_REF" \
+          --current "$HEAD_REF" \
           --output benchmark_report.md \
           --keep-results
+      continue-on-error: true
 
     # Upload benchmark results as artifacts
     - name: Upload benchmark results
@@ -54,7 +60,8 @@ jobs:
         name: benchmark-results
         path: |
           benchmark_report.md
-          benchmark_results.json
+          base_benchmark_results.json
+          current_benchmark_results.json
         retention-days: 30
 
     # Comment on PR with benchmark results.
@@ -137,9 +144,10 @@ jobs:
             console.error('Failed to read benchmark report:', e.message);
             report = `## Performance Benchmark Report
 
-          > **Failed to generate benchmark report**
+          > **Failed to generate detailed benchmark comparison**
           >
-          > This might be because:
+          > The benchmark comparison failed to run. This might be because:
+          > - Benchmark tests don't exist on the base branch yet
           > - Dependencies are missing
           > - Test execution failed
           >

From 38a82bfea3ab24a3ee0cfaff5dcb1eb210574f76 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 31 Aug 2026 00:20:51 -0700
Subject: [PATCH 55/78] perf(test): drop zookeeper from the kafka test, use a
 KRaft broker

test_module_kafka was the #2 slot in the suite at 101.8s, but the local
warm run is 13.2s. The gap is image pull: CI has no pre-pull step, so
docker run pulls implicitly on every run, and the test started two
containers totalling ~785MB (wurstmeister/kafka 468MB plus zookeeper:3.9
317MB).

Kafka has not needed zookeeper since KRaft went GA. One
apache/kafka-native:4.1.2 broker at ~150MB replaces both, so the cold
pull drops by ~635MB and the second container bring-up disappears
entirely.

Readiness was also wrong. wait_for_port_open plus a fixed sleep(1) only
proves docker bound the port, which it does at container create time
while the broker is still booting; the fixed sleep was papering over
that race. Now it probes an actual produce round-trip via the existing
wait_for_container helper, on a separate bbot_readiness topic so the
topic under assertion is untouched.

Measured, same machine, images removed first so both paths pull cold:
  before 207.7s -> after 57.5s (3.6x)
Warm: 13.2s -> 4.3s. check() and its assertion are byte identical.
Verified the failure path stays bounded: against a dead broker it raises
in 10.1s rather than hanging.
---
 .../module_tests/test_module_kafka.py         | 51 ++++++++++++-------
 1 file changed, 34 insertions(+), 17 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_kafka.py b/bbot/test/test_step_2/module_tests/test_module_kafka.py
index 2822981566..8a7ac6dc37 100644
--- a/bbot/test/test_step_2/module_tests/test_module_kafka.py
+++ b/bbot/test/test_step_2/module_tests/test_module_kafka.py
@@ -1,6 +1,8 @@
 import json
 import asyncio
 
+from bbot.test.worker import wait_for_container
+
 from .base import ModuleTestBase
 
 
@@ -16,34 +18,51 @@ class TestKafka(ModuleTestBase):
     skip_distro_tests = True
 
     async def setup_before_prep(self, module_test):
-        # Start Zookeeper
-        await self.start_container("bbot-test-zookeeper", "-p", "2181:2181", "zookeeper:3.9")
-
-        # Wait for Zookeeper to be ready
-        await self.wait_for_port_open(2181)
-
-        # Start Kafka using wurstmeister/kafka
+        # KRaft mode: one broker, no zookeeper. The native image is ~150MB
+        # against ~785MB for wurstmeister/kafka plus zookeeper, and CI pulls
+        # cold on every run.
         await self.start_container(
             "bbot-test-kafka",
-            "--link",
-            "bbot-test-zookeeper:zookeeper",
             "-e",
-            "KAFKA_ZOOKEEPER_CONNECT=zookeeper:2181",
+            "KAFKA_NODE_ID=1",
             "-e",
-            "KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092",
+            "KAFKA_PROCESS_ROLES=broker,controller",
+            "-e",
+            "KAFKA_LISTENERS=PLAINTEXT://0.0.0.0:9092,CONTROLLER://0.0.0.0:9093",
             "-e",
             "KAFKA_ADVERTISED_LISTENERS=PLAINTEXT://localhost:9092",
             "-e",
+            "KAFKA_CONTROLLER_LISTENER_NAMES=CONTROLLER",
+            "-e",
+            "KAFKA_LISTENER_SECURITY_PROTOCOL_MAP=CONTROLLER:PLAINTEXT,PLAINTEXT:PLAINTEXT",
+            "-e",
+            "KAFKA_CONTROLLER_QUORUM_VOTERS=1@localhost:9093",
+            "-e",
             "KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR=1",
+            "-e",
+            "KAFKA_TRANSACTION_STATE_LOG_REPLICATION_FACTOR=1",
+            "-e",
+            "KAFKA_TRANSACTION_STATE_LOG_MIN_ISR=1",
+            "-e",
+            "KAFKA_GROUP_INITIAL_REBALANCE_DELAY_MS=0",
             "-p",
             "9092:9092",
-            "wurstmeister/kafka",
+            "apache/kafka-native:4.1.2",
         )
 
-        # Wait for Kafka to be ready
-        await self.wait_for_port_open(9092)
+        from aiokafka import AIOKafkaProducer
+
+        # an open port is not a usable broker; probe a real produce round-trip.
+        # separate topic so the one under assertion stays untouched.
+        async def connect():
+            producer = AIOKafkaProducer(bootstrap_servers="localhost:9092")
+            await producer.start()
+            try:
+                await producer.send_and_wait("bbot_readiness", b"probe")
+            finally:
+                await producer.stop()
 
-        await asyncio.sleep(1)
+        await wait_for_container("Kafka", connect)
 
     async def check(self, module_test, events):
         from aiokafka import AIOKafkaConsumer
@@ -84,6 +103,4 @@ async def _consume():
             # Clean up: Stop the Kafka consumer
             if hasattr(self, "consumer") and not self.consumer._closed:
                 await self.consumer.stop()
-            # Stop Kafka and Zookeeper containers
             await self.stop_container("bbot-test-kafka")
-            await self.stop_container("bbot-test-zookeeper")

From c2be14db53f996187a5fabc6426c59a182837197 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 31 Aug 2026 00:54:31 -0700
Subject: [PATCH 56/78] fix(test): stop kafka readiness probe importing an
 uninstalled dep

38a82bfea replaced the port wait with an aiokafka produce round-trip to
close a real readiness race. That probe lives in setup_before_prep, which
runs BEFORE scan._prep(), and _prep() is what pip-installs a module's
deps_pip. On CI aiokafka is therefore not importable yet, so every one of
the 5 python jobs errored with ModuleNotFoundError at collection of
TestKafka. It passed locally only because the dep was already in the venv.

Go back to wait_for_port_open(9092). That is sufficient here: is_port_open
only reports ready once a connection survives its settle window, and the
KRaft broker opens its listener as the last step of boot, so the docker
proxy race that motivated the produce probe is already covered.

Measured against the real broker, probing at the instant each method
reports ready and immediately attempting a real produce: port_open 1.39s,
api_versions 0.87s, metadata 0.94s, all three followed by PRODUCE OK. Held
0/5 failures with the port probe under 2x-cpu load. The consumer in check()
still imports aiokafka, which is correct, that runs after _prep().

Verified with a scoped control that blocks aiokafka for exactly the
setup_before_prep phase, reproducing the CI condition: pre-fix tree fails
with the identical ModuleNotFoundError, post-fix tree succeeds in 2.41s.
Test passes 3/3 consecutively at ~5.7s. No assertions touched.
---
 .../module_tests/test_module_kafka.py         | 21 ++++++-------------
 1 file changed, 6 insertions(+), 15 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_kafka.py b/bbot/test/test_step_2/module_tests/test_module_kafka.py
index 8a7ac6dc37..6f05c3d14d 100644
--- a/bbot/test/test_step_2/module_tests/test_module_kafka.py
+++ b/bbot/test/test_step_2/module_tests/test_module_kafka.py
@@ -1,8 +1,6 @@
 import json
 import asyncio
 
-from bbot.test.worker import wait_for_container
-
 from .base import ModuleTestBase
 
 
@@ -50,19 +48,12 @@ async def setup_before_prep(self, module_test):
             "apache/kafka-native:4.1.2",
         )
 
-        from aiokafka import AIOKafkaProducer
-
-        # an open port is not a usable broker; probe a real produce round-trip.
-        # separate topic so the one under assertion stays untouched.
-        async def connect():
-            producer = AIOKafkaProducer(bootstrap_servers="localhost:9092")
-            await producer.start()
-            try:
-                await producer.send_and_wait("bbot_readiness", b"probe")
-            finally:
-                await producer.stop()
-
-        await wait_for_container("Kafka", connect)
+        # this runs before _prep(), which is what pip-installs the module's
+        # deps, so nothing here may import aiokafka. The broker opens its own
+        # listener as the last step of boot, after "Enabling request
+        # processing", so a connection that survives is_port_open's settle
+        # window is a ready broker rather than the docker proxy.
+        await self.wait_for_port_open(9092)
 
     async def check(self, module_test, events):
         from aiokafka import AIOKafkaConsumer

From a49a86d8a26e432e9263eefb910b32ab77b6de44 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 31 Aug 2026 08:45:01 -0700
Subject: [PATCH 57/78] fix(ci): let pytest-timeout cover fixture setup and
 teardown

timeout_func_only=true scopes the 1200s timeout to the test body only, so
time spent in fixture setup and teardown is never counted. That is exactly
where this suite's worst wedges have happened: the websocket hang (2f4032862)
burned 6h0m18s and was killed by the workflow limit, not by pytest, because
it wedged in fixture teardown cancelling orphaned tasks. The orphan-cancel
bound (52e41fa65) fixed that specific hang, but the blind spot itself is
still open, and every module test does its real work in the module_test
fixture rather than in the body.

Measured the semantics directly rather than trusting the flag name. With
timeout_func_only=true and --timeout 3: an 8s sleep in fixture setup passes,
an 8s sleep in teardown passes, only a body sleep is caught. With
timeout_func_only=false all three are caught, setup and teardown as ERROR.

The timeout also becomes cumulative across phases, so the budget has to
clear the slowest whole test, not just its body. Confirmed with a 2s setup
plus 2s body against --timeout 3: passes under the old setting, trips under
the new one. Worst real test across all five green py jobs is test_cli_args
at 150.9s to 189.6s wall including setup, against a 1200s budget, so the
margin is 6.3x. Nothing is near the line.

Verified: bbot/test/test_step_1/ under the new setting, 298 passed, zero
timeouts. badsecrets plus webbrute_shortnames plus modules_basic, 11 passed.
ruff check and ruff format clean.

No test logic touched, no assertions changed, nothing split.
---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index d81333f76b..f0a3cf30a2 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -38,7 +38,7 @@ jobs:
         env:
           BBOT_IO_API_KEY: ${{ secrets.BBOT_IO_API_KEY }}
         run: |
-          uv run pytest -vv -n $(python bbot/test/worker_count.py) --dist loadgroup --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+          uv run pytest -vv -n $(python bbot/test/worker_count.py) --dist loadgroup --reruns 2 -o timeout_func_only=false --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v7

From 1e3d27186379c2187ef24b1955da3c504d53b247 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 31 Aug 2026 09:24:29 -0700
Subject: [PATCH 58/78] perf(scanner): stop compiling throwaway regexes for the
 YARA dns rules

dns_regexes_yara compiled a Python regex object per target and then only
ever read .pattern back off it. The compiled objects were never matched
against anything: their single consumer, dns_yara_rules_uncompiled, splices
the source string into YARA rule text, and YARA does its own compilation.
So every target paid a full regex compile whose only product was the string
that went into it.

That is invisible on a normal scan and brutal on a big one. Profiled
test_huge_target_list (10,005 targets) under cProfile: scan._prep spent
16.3s in excavate setup, 16.1s of that in _generate_dns_regexes, and 12.4s
of that inside regex._compile for 10,032 patterns that were immediately
discarded.

Split the generation in two. _generate_dns_regex_patterns builds the source
strings, _generate_dns_regexes compiles them for dns_regexes (which really
does match, via oauth.py and helpers.re.findall_multi), and the YARA
property now takes the uncompiled strings.

Equivalence is exact, not approximate: the old path emitted
re.compile(p, re.I).pattern and the new one emits p. Verified those are
byte-identical across 10,013 targets including punycode, underscores,
dashes, mixed case, IPs and multi-label public suffixes. Rule text for the
single, multi and huge target shapes is unchanged, and dns_regexes still
yields compiled objects (10,005 of them, all with .finditer).

Note the rule-dict hash is not stable run to run, before or after this
change: dns_strings derives from a set, so ordering varies and the
$dns_name_N numbering shifts with it. Confirmed pre-existing by hashing
twice on the unmodified tree. Not introduced here.

test_huge_target_list 7.65s -> 3.36s. It sits on gw2, the binding worker
in the py3.13 job (599s busy, 0s slack), so this comes off the critical
path rather than off a worker that was already idle.

Verified: test_step_1 302 passed, test_module_excavate 52 passed,
test_regexes 6 passed. ruff check and ruff format clean.
No assertions changed, no tests split.
---
 bbot/scanner/scanner.py | 25 ++++++++++++++++++-------
 1 file changed, 18 insertions(+), 7 deletions(-)

diff --git a/bbot/scanner/scanner.py b/bbot/scanner/scanner.py
index fc1f7c4765..93c9baa0db 100644
--- a/bbot/scanner/scanner.py
+++ b/bbot/scanner/scanner.py
@@ -1216,6 +1216,18 @@ def dns_strings(self):
             self._dns_strings = dns_strings
         return self._dns_strings
 
+    def _generate_dns_regex_patterns(self, pattern):
+        """
+        Generates a list of DNS hostname regex source strings based on the provided pattern.
+
+        Args:
+            pattern (str):
+        Returns:
+            list[str]: A list of regex source strings.
+        """
+
+        return [f"{pattern}{re.escape(t)})" for t in self.dns_strings]
+
     def _generate_dns_regexes(self, pattern):
         """
         Generates a list of compiled DNS hostname regexes based on the provided pattern.
@@ -1228,10 +1240,9 @@ def _generate_dns_regexes(self, pattern):
         """
 
         dns_regexes = []
-        for t in self.dns_strings:
-            regex_pattern = re.compile(f"{pattern}{re.escape(t)})", re.I)
-            log.debug(f"Generated Regex [{regex_pattern.pattern}] for domain {t}")
-            dns_regexes.append(regex_pattern)
+        for regex_pattern in self._generate_dns_regex_patterns(pattern):
+            log.debug(f"Generated Regex [{regex_pattern}]")
+            dns_regexes.append(re.compile(regex_pattern, re.I))
         return dns_regexes
 
     @property
@@ -1253,10 +1264,10 @@ def dns_regexes(self):
     @property
     def dns_regexes_yara(self):
         """
-        Returns a list of DNS hostname regexes formatted specifically for compatibility with YARA rules.
+        Returns a list of DNS hostname regex source strings formatted specifically for compatibility with YARA rules.
         """
         if self._dns_regexes_yara is None:
-            self._dns_regexes_yara = self._generate_dns_regexes(r"(([a-z0-9-]+\.)*")
+            self._dns_regexes_yara = self._generate_dns_regex_patterns(r"(([a-z0-9-]+\.)*")
         return self._dns_regexes_yara
 
     @property
@@ -1264,7 +1275,7 @@ def dns_yara_rules_uncompiled(self):
         if self._dns_yara_rules_uncompiled is None:
             regexes_component_list = []
             for i, r in enumerate(self.dns_regexes_yara):
-                regexes_component_list.append(rf"$dns_name_{i} = /\b{r.pattern}/ nocase")
+                regexes_component_list.append(rf"$dns_name_{i} = /\b{r}/ nocase")
 
             # Chunk the regexes into groups of 10,000
             chunk_size = 10000

From 07cd939a8face39926f7ef0eee111566403e02ac Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 31 Aug 2026 09:25:27 -0700
Subject: [PATCH 59/78] revert(ci): restore timeout_func_only to unblock the
 push

Reverts a49a86d8a. Not a correctness retreat: that commit is still right,
and the reasoning and measurements in its message stand.

The push credential is a gh OAuth token with scopes gist, project,
read:org, repo and no workflow scope, so any push whose net diff touches
.github/workflows is rejected outright. a49a86d8a edits tests.yml, and the
perf work now sits on top of it, so the workflow commit blocks a change
that has nothing to do with workflows. Confirmed GitHub judges the net
tree diff rather than per-commit: with this revert on top the same push
is accepted.

Reverting rather than reordering because rebase is off limits on this
branch, and this keeps a49a86d8a and its evidence in history. Same
precedent as 3c1125d8d for benchmark.yml.

TO RESTORE (one time, needs a human at a browser):
  gh auth refresh -h github.com -s workflow
  git revert 6ed6e058e && git push origin perf/ci-suite-walltime
---
 .github/workflows/tests.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
index f0a3cf30a2..d81333f76b 100644
--- a/.github/workflows/tests.yml
+++ b/.github/workflows/tests.yml
@@ -38,7 +38,7 @@ jobs:
         env:
           BBOT_IO_API_KEY: ${{ secrets.BBOT_IO_API_KEY }}
         run: |
-          uv run pytest -vv -n $(python bbot/test/worker_count.py) --dist loadgroup --reruns 2 -o timeout_func_only=false --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
+          uv run pytest -vv -n $(python bbot/test/worker_count.py) --dist loadgroup --reruns 2 -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO --cov-config=bbot/test/coverage.cfg --cov-report xml:cov.xml --cov=bbot .
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v7

From 11a2551b44f77d17cbb05df2717b02dfd5c31c13 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 31 Aug 2026 12:06:22 -0700
Subject: [PATCH 60/78] perf(webbrute_shortnames): index the prediction models
 by prefix

MinimalWordPredictor.predict() linear-scanned every entry in the model,
calling str.startswith on all of them, then sorted the full match list.
Each loaded model holds ~2.1M words, so a single predict() cost ~165ms
regardless of how few words could possibly match.

test_module_webbrute_shortnames makes 30 predict() calls, one per hint
per fuzz strategy. Measured inside the scan: 9.20s of the 24.7s scan was
predict(), and cProfile attributed 63M startswith calls and 12.4s of
tottime to that one comprehension.

Now each model is bucketed once by the first two characters of every
word, and predict() scans only the matching bucket. Prefixes shorter
than the key length fall back to the full scan, so behavior is unchanged
for them. The index is built eagerly in setup() rather than lazily,
because predict() runs in the cpu executor and concurrent callers would
otherwise each rebuild it. Class-level defaults cover the unpickled
instances, which never run __init__.

Output is exactly identical, not merely equivalent: 3222 comparisons
across both real models (the module test's own prefixes, edge cases
including empty, non-ascii, uppercase and punctuation, plus 500 sampled
prefixes) at top_n 250/25/1 produce byte-identical result lists. Ties
preserve their old ordering because the sort remains a stable sort on
frequency over a candidate set kept in model insertion order.

predict() 9.20s -> 1.72s, index build 0.48s and 0.53s.
Test wall time 28.4s -> 25.4s locally.
Verified: 14 passed across test_module_webbrute_shortnames,
test_module_iis_shortnames and test_module_webbrute. ruff clean.
---
 bbot/modules/webbrute_shortnames.py | 28 +++++++++++++++++++++++++++-
 1 file changed, 27 insertions(+), 1 deletion(-)

diff --git a/bbot/modules/webbrute_shortnames.py b/bbot/modules/webbrute_shortnames.py
index 0883c22d3c..9bd52367a2 100644
--- a/bbot/modules/webbrute_shortnames.py
+++ b/bbot/modules/webbrute_shortnames.py
@@ -47,6 +47,8 @@ class Config(BaseModuleConfig):
 
     supplementary_words = ["html", "ajax", "xml", "json", "api"]
 
+    prefix_index_length = 2
+
     async def generate_templist(self, hint, shortname_type):
         words = await self.helpers.run_in_executor_cpu(self._generate_templist_sync, hint, shortname_type)
         return words, len(words)
@@ -103,13 +105,32 @@ async def setup(self):
         self.rate = self.config.get("rate", 0) or None
         self.concurrency = 50
 
+        prefix_index_length = self.prefix_index_length
+
         class MinimalWordPredictor:
+            # class-level defaults: unpickling restores state without calling __init__
+            prefix_index = {}
+            prefix_index_length = 0
+
             def __init__(self):
                 self.word_frequencies = {}
+                self.prefix_index = {}
+                self.prefix_index_length = 0
+
+            def build_prefix_index(self, prefix_length):
+                index = {}
+                for word in self.word_frequencies:
+                    index.setdefault(word[:prefix_length], []).append(word)
+                self.prefix_index = index
+                self.prefix_index_length = prefix_length
 
             def predict(self, prefix, top_n):
                 prefix = prefix.lower()
-                matches = [(word, freq) for word, freq in self.word_frequencies.items() if word.startswith(prefix)]
+                if self.prefix_index and len(prefix) >= self.prefix_index_length:
+                    candidates = self.prefix_index.get(prefix[: self.prefix_index_length], ())
+                    matches = [(word, self.word_frequencies[word]) for word in candidates if word.startswith(prefix)]
+                else:
+                    matches = [(word, freq) for word, freq in self.word_frequencies.items() if word.startswith(prefix)]
 
                 if not matches:
                     return []
@@ -146,6 +167,11 @@ def find_class(self, module, name):
             unpickler = CustomUnpickler(f)
             self.directory_predictor = unpickler.load()
 
+        # built once here, not lazily: predict() runs concurrently in the cpu
+        # executor, and a lazy build would be repeated by every racing caller
+        for predictor in (self.endpoint_predictor, self.directory_predictor):
+            await self.helpers.run_in_executor_cpu(predictor.build_prefix_index, prefix_index_length)
+
         self.subword_list = []
         if self.find_subwords:
             self.debug("Acquiring shortname subword list")

From 0d930182bb2e27dbbc7efbbd7ebe43c909cbbcd8 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 31 Aug 2026 13:55:37 -0700
Subject: [PATCH 61/78] perf(nuclei): exclude unmatchable tcp templates from
 the technology test

TestNucleiTechnology ran 21.9s locally / 38s in CI. Traced the whole cost to a
single nuclei subprocess (19.4s of the 21.9s), then timestamped nuclei's own
stderr phases against a standalone apache-flavoured server:

   0.4  280 templates loaded
   1.0  automatic scan clustered, begins
   6.1  tech-detect finishes (the phase the test actually asserts on)
  21.3  scan completed

Three 5s stalls account for 15s of the 21s: kafka-topics-list (9092) twice and
CVE-2021-44521 (cassandra 9042), each dialing a closed port and waiting out a
read timeout. The fixture is a pytest-httpserver on one HTTP port, so no
tcp-protocol template can ever match. That time buys zero coverage.

Corrects two findings from the previous audit run:

1. Narrowing `templates` does NOT help. TestNucleiTechnology runs in `-as`
   (automatic scan) mode, where nuclei selects templates from wappalyzer
   fingerprints and ignores `-templates` for execution. Measured: loading 38
   templates instead of 280 still executes 280 and still takes 22.2s.
2. The earlier 73s baseline was measured at `-rate-limit 10`, inherited from
   the parent class. TestNucleiTechnology does not inherit that override, so
   it runs at the 150 default. Real baseline is 21-22s, not 73s.

Adds `etypes` as a real module option mapping to nuclei's `-exclude-type`,
defaulting to "" so behavior is unchanged for every existing config. The test
opts into `etypes: tcp`.

Result: TestNucleiTechnology 21.9s -> 7.6s, whole file 78.8s -> 62.1s, 9 passed
both before and after. The TECHNOLOGY assertion is untouched and still passes,
since tech-detect completes at t=6.1 well before the excluded templates ran.

Also passes `-timeout self.scan.http_timeout`. bbot never sent `-timeout`, so
nuclei silently used its own 10s default and `web.http_timeout` was ignored by
this module alone while http, webbrute, telerik and ntlm all honour it. This is
a config-fidelity fix, NOT a perf win: A/B at 3 runs each showed no wall-time
effect (default 23.7/23.7/23.8s vs -timeout 3 at 23.6/23.7/23.7s) because the
tcp dialers do not observe it. The default value passed is 10, identical to
nuclei's own default, so nothing changes unless the user set http_timeout.

Both flags are pinned by asserting on the captured command line. Verified
negatively: dropping either flag from the builder fails the test.
---
 bbot/modules/nuclei.py                        | 15 +++++++++++
 .../module_tests/test_module_nuclei.py        | 27 ++++++++++++++++++-
 docs/modules/nuclei.md                        |  1 +
 docs/scanning/configuration.md                |  1 +
 4 files changed, 43 insertions(+), 1 deletion(-)

diff --git a/bbot/modules/nuclei.py b/bbot/modules/nuclei.py
index 877ee14cfb..fab3114cba 100644
--- a/bbot/modules/nuclei.py
+++ b/bbot/modules/nuclei.py
@@ -46,6 +46,13 @@ class Config(BaseModuleConfig):
             ),
         )
         etags: str = Field("", description="tags to exclude from the scan")
+        etypes: str = Field(
+            "",
+            description=(
+                "protocol types to exclude from the scan, comma-separated "
+                "(dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code, javascript)"
+            ),
+        )
         budget: int = Field(1, description="Used in budget mode to set the number of allowed requests per host")
         silent: bool = Field(False, description="Don't display nuclei's banner or status messages")
         directory_only: bool = Field(True, description="Filter out 'file' URL event (default True)")
@@ -105,6 +112,9 @@ async def setup(self):
         self.etags = self.config.get("etags")
         if self.etags:
             self.info(f"Excluding the following nuclei tags: [{self.etags}]")
+        self.etypes = self.config.get("etypes")
+        if self.etypes:
+            self.info(f"Excluding the following nuclei protocol types: [{self.etypes}]")
         self.severity = self.config.get("severity")
         if self.mode != "severe" and self.severity != "":
             self.info(f"Limiting nuclei templates to the following severities: [{self.severity}]")
@@ -233,6 +243,8 @@ async def execute_nuclei(self, nuclei_input):
             "-stats-json",
             "-retries",
             self.retries,
+            "-timeout",
+            self.scan.http_timeout,
         ]
 
         if self.helpers.system_resolvers:
@@ -248,6 +260,9 @@ async def execute_nuclei(self, nuclei_input):
                 command.append(f"-{cli_option}")
                 command.append(option)
 
+        if self.etypes:
+            command += ["-exclude-type", self.etypes]
+
         if self.scan.config.get("interactsh_disable") is True:
             self.info("Disabling interactsh in accordance with global settings")
             command.append("-no-interactsh")
diff --git a/bbot/test/test_step_2/module_tests/test_module_nuclei.py b/bbot/test/test_step_2/module_tests/test_module_nuclei.py
index f139933e9f..ffe6584b4b 100644
--- a/bbot/test/test_step_2/module_tests/test_module_nuclei.py
+++ b/bbot/test/test_step_2/module_tests/test_module_nuclei.py
@@ -90,9 +90,13 @@ def check(self, module_test, events):
 
 
 class TestNucleiTechnology(TestNucleiManual):
+    # etypes tcp: the fixture is a single HTTP port, so tcp-protocol templates
+    # (cassandra 9042, kafka 9092, ajp 8009) can only ever dial a closed port and
+    # wait out nuclei's fixed 5s read timeout. They match nothing here.
     config_overrides = {
+        "web": {"http_timeout": 7},
         "interactsh_disable": True,
-        "modules": {"nuclei": {"mode": "technology", "concurrency": 2, "tags": "apache"}},
+        "modules": {"nuclei": {"mode": "technology", "concurrency": 2, "tags": "apache", "etypes": "tcp"}},
     }
 
     async def setup_before_prep(self, module_test):
@@ -103,10 +107,31 @@ async def setup_before_prep(self, module_test):
         }
         module_test.set_expect_requests(expect_args=expect_args, respond_args=respond_args)
 
+    async def setup_after_prep(self, module_test):
+        self.commands = []
+        module = module_test.scan.modules["nuclei"]
+        original = module.run_process_live
+
+        def capture(*args, **kwargs):
+            command = args[0] if args and isinstance(args[0], (list, tuple)) else args
+            self.commands.append([str(c) for c in command])
+            return original(*args, **kwargs)
+
+        module.run_process_live = capture
+
     def check(self, module_test, events):
         assert any(e.type == "TECHNOLOGY" and "apache" in e.data["technology"].lower() for e in events)
         assert "Using Interactsh Server" not in open(module_test.scan.home / "debug.log").read()
 
+        assert self.commands, "nuclei was never executed"
+        for command in self.commands:
+            assert "-exclude-type" in command, f"-exclude-type missing from {command}"
+            assert command[command.index("-exclude-type") + 1] == "tcp"
+            # nuclei silently defaults to its own 10s, so an unpassed -timeout
+            # means web.http_timeout is ignored by this module alone
+            assert "-timeout" in command, f"-timeout missing from {command}"
+            assert command[command.index("-timeout") + 1] == "7"
+
 
 class TestNucleiBudget(TestNucleiManual):
     config_overrides = {
diff --git a/docs/modules/nuclei.md b/docs/modules/nuclei.md
index f9f9075c14..82c8ed0107 100644
--- a/docs/modules/nuclei.md
+++ b/docs/modules/nuclei.md
@@ -43,6 +43,7 @@ The Nuclei module has many configuration options:
 | modules.nuclei.concurrency    | int                                                 | maximum number of templates to be executed in parallel (default 25)                                                                                                                                                                                                                                                            | 25        |
 | modules.nuclei.directory_only | bool                                                | Filter out 'file' URL event (default True)                                                                                                                                                                                                                                                                                     | True      |
 | modules.nuclei.etags          | str                                                 | tags to exclude from the scan                                                                                                                                                                                                                                                                                                  |           |
+| modules.nuclei.etypes         | str                                                 | protocol types to exclude from the scan, comma-separated (dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code, javascript)                                                                                                                                                                                   |           |
 | modules.nuclei.mode           | Literal['manual', 'technology', 'severe', 'budget'] | manual &#124; technology &#124; severe &#124; budget. Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). Manual (DEFAULT): Fully manual settings. Severe: Only critical and high severity templates without intrusive. Budget: Limit Nuclei to a specified number of HTTP requests | manual    |
 | modules.nuclei.module_timeout | int                                                 | Max time in seconds to spend handling each batch of events                                                                                                                                                                                                                                                                     | 21600     |
 | modules.nuclei.ratelimit      | int                                                 | maximum number of requests to send per second (default 150)                                                                                                                                                                                                                                                                    | 150       |
diff --git a/docs/scanning/configuration.md b/docs/scanning/configuration.md
index b392fbdb7f..6aae71afcd 100644
--- a/docs/scanning/configuration.md
+++ b/docs/scanning/configuration.md
@@ -519,6 +519,7 @@ In addition to the stated options for each module, the following universal optio
 | modules.nuclei.concurrency                             | int                                                  | maximum number of templates to be executed in parallel (default 25)                                                                                                                                                                                                                                                            | 25                                                                                                                                                                                                                                                                                                                                                                                                                                                      |
 | modules.nuclei.directory_only                          | bool                                                 | Filter out 'file' URL event (default True)                                                                                                                                                                                                                                                                                     | True                                                                                                                                                                                                                                                                                                                                                                                                                                                    |
 | modules.nuclei.etags                                   | str                                                  | tags to exclude from the scan                                                                                                                                                                                                                                                                                                  |                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| modules.nuclei.etypes                                  | str                                                  | protocol types to exclude from the scan, comma-separated (dns, file, http, headless, tcp, workflow, ssl, websocket, whois, code, javascript)                                                                                                                                                                                   |                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
 | modules.nuclei.mode                                    | Literal['manual', 'technology', 'severe', 'budget']  | manual &#124; technology &#124; severe &#124; budget. Technology: Only activate based on technology events that match nuclei tags (nuclei -as mode). Manual (DEFAULT): Fully manual settings. Severe: Only critical and high severity templates without intrusive. Budget: Limit Nuclei to a specified number of HTTP requests | manual                                                                                                                                                                                                                                                                                                                                                                                                                                                  |
 | modules.nuclei.module_timeout                          | int                                                  | Max time in seconds to spend handling each batch of events                                                                                                                                                                                                                                                                     | 21600                                                                                                                                                                                                                                                                                                                                                                                                                                                   |
 | modules.nuclei.ratelimit                               | int                                                  | maximum number of requests to send per second (default 150)                                                                                                                                                                                                                                                                    | 150                                                                                                                                                                                                                                                                                                                                                                                                                                                     |

From c426540411bf47e1f4291ac2f7a3d445ae95499c Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 31 Aug 2026 15:06:06 -0700
Subject: [PATCH 62/78] fix(nuclei): stop a failed template fetch from
 cascading into 66 downloads

The py3.14 job failed with all 9 nuclei tests erroring in scan._prep() on
"Failed to install nuclei templates after retry". The log shows 66 template
update attempts, 0 successes, 33 wipes and 33 hard failures for what should
have been a single install. py3.10 through 3.13 passed on the same commit.

Two defects in the setup() repair path, both amplifiers rather than the
originating fetch failure:

The repair branch treated "no templates on disk" as the stale-marker
corruption it was written for, so a plain download failure took the wipe and
re-download path. nuclei had produced no files, so there was nothing corrupt
to repair, and the retry just doubled the request count against a source that
was already failing. Every one of the 33 module setups paid two downloads
instead of one. _run_template_update now returns its classified outcome and
the repair is skipped when the update reported failure, so a bad fetch fails
after one attempt.

The wipe also ran outside the template lock that 79c361a32 added, while the
update it guards runs inside. shutil.rmtree removes the whole nuclei-state
dir, so a worker entering repair could delete the tree another worker was
mid-extract into, turning one worker's failure into its neighbours'. The
update, verify, wipe and retry sequence now lives in a single _ensure_templates
that holds the lock across all four. The lock file sits in tools_dir, not
under nuclei-state, so the wipe cannot destroy it.

The healthy path is unchanged: an update that lands templates still returns
before any repair, and the stale-marker case still wipes and retries exactly
once.

Both behaviours are regression-pinned and verified negatively. Restoring the
wipe outside the lock fails test_nuclei_repair_wipe_holds_the_lock with "state
dir was wiped without holding the template lock"; removing the fail-fast guard
fails test_nuclei_failed_download_does_not_wipe_and_refetch with "a failed
download must not trigger a second fetch". No assertions were removed or
weakened. Full file: 11 passed, up from 9.
---
 bbot/modules/nuclei.py                        |  39 ++++--
 .../module_tests/test_module_nuclei.py        | 127 ++++++++++++++++++
 2 files changed, 152 insertions(+), 14 deletions(-)

diff --git a/bbot/modules/nuclei.py b/bbot/modules/nuclei.py
index fab3114cba..ff314b7730 100644
--- a/bbot/modules/nuclei.py
+++ b/bbot/modules/nuclei.py
@@ -85,18 +85,8 @@ async def setup(self):
         self.nuclei_templates_dir = self.nuclei_state_dir / "templates"
         self.nuclei_config_dir.mkdir(parents=True, exist_ok=True)
         self.nuclei_cache_dir.mkdir(parents=True, exist_ok=True)
-        await self._update_templates()
-        # nuclei writes its version marker before the tarball finishes extracting,
-        # so a killed update can leave the marker pointing at an empty dir and
-        # every subsequent run reports "up-to-date." Verify and repair once.
-        if not self._templates_installed():
-            self.warning("Nuclei templates appear incomplete; wiping isolated state and re-downloading")
-            shutil.rmtree(self.nuclei_state_dir, ignore_errors=True)
-            self.nuclei_config_dir.mkdir(parents=True, exist_ok=True)
-            self.nuclei_cache_dir.mkdir(parents=True, exist_ok=True)
-            await self._update_templates()
-            if not self._templates_installed():
-                return False, "Failed to install nuclei templates after retry"
+        if not await self._ensure_templates():
+            return False, "Failed to install nuclei templates after retry"
         self.proxy = self.scan.web_config.get("http_proxy", "")
         self.mode = self.config.get("mode")
         self.ratelimit = self.config.get("ratelimit")
@@ -367,17 +357,37 @@ def _nuclei_env(self):
         env["XDG_CACHE_HOME"] = str(self.nuclei_cache_dir)
         return env
 
-    async def _update_templates(self):
+    async def _ensure_templates(self):
         # 13k+ template files extract into one shared dir. Concurrent updaters
         # (xdist workers, parallel scans) fight over the same tree and each
         # re-does the other's work, so serialize on a lock beside it. A waiter
         # that finds the tree already populated skips its own redundant update.
+        # The corruption repair below wipes the tree, so it has to run under the
+        # same lock: wiping while another worker is mid-extract destroys its
+        # output and makes it fail too.
         uncontended = await self.helpers.run_in_executor_io(self._acquire_template_lock)
         try:
             if not uncontended and self._templates_installed():
                 self.info("Nuclei templates already up-to-date")
-                return
+                return True
+            outcome = await self._run_template_update()
+            if self._templates_installed():
+                return True
+            # A download that never produced files is a fetch failure, not the
+            # stale-marker corruption the wipe exists to repair. Wiping and
+            # re-downloading just doubles the requests against a source that is
+            # already failing, so only repair when the update claimed success.
+            if outcome == "failure":
+                return False
+            # nuclei writes its version marker before the tarball finishes
+            # extracting, so a killed update can leave the marker pointing at an
+            # empty dir and every subsequent run reports "up-to-date."
+            self.warning("Nuclei templates appear incomplete; wiping isolated state and re-downloading")
+            shutil.rmtree(self.nuclei_state_dir, ignore_errors=True)
+            self.nuclei_config_dir.mkdir(parents=True, exist_ok=True)
+            self.nuclei_cache_dir.mkdir(parents=True, exist_ok=True)
             await self._run_template_update()
+            return self._templates_installed()
         finally:
             self._release_template_lock()
 
@@ -418,6 +428,7 @@ async def _run_template_update(self):
             self.info("Nuclei templates already up-to-date")
         else:
             self.warning(f"Failure while updating nuclei templates: {update_results.stderr or '<no stderr>'}")
+        return outcome
 
     # nuclei's success messaging has drifted across releases (installed / updated /
     # downloaded). Match any of them so a future rename doesn't silently downgrade
diff --git a/bbot/test/test_step_2/module_tests/test_module_nuclei.py b/bbot/test/test_step_2/module_tests/test_module_nuclei.py
index ffe6584b4b..5885787c91 100644
--- a/bbot/test/test_step_2/module_tests/test_module_nuclei.py
+++ b/bbot/test/test_step_2/module_tests/test_module_nuclei.py
@@ -2,6 +2,10 @@
 from .base import ModuleTestBase
 from bbot.test.worker import HTTPSERVER_URL, BBOT_TEST_DIR, BBOT_TEST_TOOLS_DIR
 
+import fcntl
+from types import SimpleNamespace
+from unittest.mock import patch
+
 
 class TestNucleiManual(ModuleTestBase):
     targets = [HTTPSERVER_URL]
@@ -258,6 +262,129 @@ def test_nuclei_classify_update_stderr():
     assert c(None) == "failure"
 
 
+@pytest.mark.asyncio
+async def test_nuclei_repair_wipe_holds_the_lock(tmp_path):
+    """Regression: the corruption repair wipes the whole nuclei state dir. If it
+    runs outside the template lock, a worker that wipes while another is
+    mid-extract destroys the other's output, so both fail and every nuclei test
+    on every xdist worker hard-fails with "Failed to install nuclei templates
+    after retry". Pin that the wipe only happens while the lock is held.
+    """
+    from bbot.modules.nuclei import nuclei
+
+    tools_dir = tmp_path / "tools"
+    tools_dir.mkdir()
+    state_dir = tools_dir / "nuclei-state"
+    templates_dir = state_dir / "templates"
+
+    mod = nuclei.__new__(nuclei)
+    mod.nuclei_state_dir = state_dir
+    mod.nuclei_config_dir = state_dir / "config"
+    mod.nuclei_cache_dir = state_dir / "cache"
+    mod.nuclei_templates_dir = templates_dir
+    mod.nuclei_config_dir.mkdir(parents=True, exist_ok=True)
+    mod.nuclei_cache_dir.mkdir(parents=True, exist_ok=True)
+
+    helpers = SimpleNamespace(
+        tools_dir=tools_dir,
+        run_in_executor_io=lambda fn, *a: asyncio.get_running_loop().run_in_executor(None, fn, *a),
+    )
+    for name in ("info", "warning", "success", "debug"):
+        setattr(mod, name, lambda *a, **kw: None)
+
+    held_during_wipe = []
+    original_rmtree = shutil.rmtree
+
+    def probe_rmtree(path, *args, **kwargs):
+        # a second process must not be able to take the lock while we wipe
+        probe = open(tools_dir / "nuclei-templates.lock", "w")
+        try:
+            try:
+                fcntl.flock(probe, fcntl.LOCK_EX | fcntl.LOCK_NB)
+                held_during_wipe.append(False)
+                fcntl.flock(probe, fcntl.LOCK_UN)
+            except OSError:
+                held_during_wipe.append(True)
+        finally:
+            probe.close()
+        return original_rmtree(path, *args, **kwargs)
+
+    # first update produces nothing (simulates the killed/incomplete extract),
+    # second one populates the tree so the repair path is exercised end to end
+    calls = []
+
+    async def fake_update():
+        calls.append(1)
+        if len(calls) > 1:
+            (templates_dir / "http").mkdir(parents=True, exist_ok=True)
+            (templates_dir / "http" / "t.yaml").write_text("id: t")
+        # claimed success but produced nothing: the stale-marker corruption the
+        # wipe exists to repair
+        return "updated"
+
+    mod._run_template_update = fake_update
+
+    with patch.object(shutil, "rmtree", probe_rmtree), patch.object(nuclei, "helpers", helpers):
+        installed = await mod._ensure_templates()
+
+    assert installed, "repair path should report success once templates land"
+    assert calls == [1, 1], "repair should run exactly one retry update"
+    assert held_during_wipe == [True], "state dir was wiped without holding the template lock"
+    # lock must be released afterwards so the next worker can proceed
+    after = open(tools_dir / "nuclei-templates.lock", "w")
+    try:
+        fcntl.flock(after, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        fcntl.flock(after, fcntl.LOCK_UN)
+    finally:
+        after.close()
+
+
+@pytest.mark.asyncio
+async def test_nuclei_failed_download_does_not_wipe_and_refetch(tmp_path):
+    """Regression: when the template download itself fails, nuclei produced no
+    files, so there is no stale-marker corruption to repair. Wiping and
+    re-downloading doubles the requests against an already-failing source. In CI
+    this turned one bad fetch into 66 downloads and 33 hard failures across every
+    xdist worker. A "failure" outcome must fail fast without a second fetch.
+    """
+    from bbot.modules.nuclei import nuclei
+
+    tools_dir = tmp_path / "tools"
+    tools_dir.mkdir()
+    state_dir = tools_dir / "nuclei-state"
+
+    mod = nuclei.__new__(nuclei)
+    mod.nuclei_state_dir = state_dir
+    mod.nuclei_config_dir = state_dir / "config"
+    mod.nuclei_cache_dir = state_dir / "cache"
+    mod.nuclei_templates_dir = state_dir / "templates"
+    mod.nuclei_config_dir.mkdir(parents=True, exist_ok=True)
+    mod.nuclei_cache_dir.mkdir(parents=True, exist_ok=True)
+
+    helpers = SimpleNamespace(
+        tools_dir=tools_dir,
+        run_in_executor_io=lambda fn, *a: asyncio.get_running_loop().run_in_executor(None, fn, *a),
+    )
+    for name in ("info", "warning", "success", "debug"):
+        setattr(mod, name, lambda *a, **kw: None)
+
+    calls = []
+    wiped = []
+
+    async def failing_update():
+        calls.append(1)
+        return "failure"
+
+    mod._run_template_update = failing_update
+
+    with patch.object(shutil, "rmtree", lambda *a, **kw: wiped.append(1)), patch.object(nuclei, "helpers", helpers):
+        installed = await mod._ensure_templates()
+
+    assert installed is False, "a failed download must not report success"
+    assert calls == [1], "a failed download must not trigger a second fetch"
+    assert wiped == [], "a failed download must not wipe state that holds no corruption"
+
+
 class TestNucleiCustomHeaders(TestNucleiManual):
     custom_headers = {"testheader1": "test1", "testheader2": "test2"}
     config_overrides = TestNucleiManual.config_overrides

From c1aa4638a30fe405e509c0a04af0c01951c3a4e6 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 31 Aug 2026 15:47:49 -0700
Subject: [PATCH 63/78] fix(excavate): stop module introspection from firing
 attribute descriptors

excavate.setup() calls find_subclasses() on every sibling module to collect
their ExcavateRules. That used inspect.getmembers(), which getattrs every
name on the object, including BaseModule's ~20 properties. One of them,
memory_usage, walks the module's own __dict__ via get_size().

setup_modules() runs all module setup() coroutines concurrently through
as_completed(), so while excavate introspects a sibling, that sibling is
still assigning its own self.<attr> in setup(). The get_size() walk then
loses the race and raises "dictionary changed size during iteration",
which hard-fails excavate for the whole scan.

Observed on py3.10 in CI (12 occurrences in one job) while 3.11-3.14 were
green: pure timing, nothing version-specific.

find_subclasses only ever wants classes, and classes live in namespace
dicts, so reading vars() off the instance and its MRO gets the same result
without invoking a single descriptor. Snapshots each namespace under retry
so the read itself cannot tear. Ordering is now name-sorted rather than
getmembers' incidental sort, and first definition wins on override, which
matches the previous MRO-flattened behavior.

Verified equivalent against the old implementation on 303 BaseModule
classes (0 mismatches), the excavate class (12 rules), and
ParameterExtractor (10 rules). Race test pinned and negatively verified:
the old path raises 53/3000 iterations, the new path 0/3000, and restoring
getmembers() fails the new test.
---
 bbot/modules/internal/excavate.py             | 37 ++++++++++++--
 .../module_tests/test_module_excavate.py      | 50 +++++++++++++++++++
 2 files changed, 82 insertions(+), 5 deletions(-)

diff --git a/bbot/modules/internal/excavate.py b/bbot/modules/internal/excavate.py
index cdc5c4ac5e..cc5fb7d7af 100644
--- a/bbot/modules/internal/excavate.py
+++ b/bbot/modules/internal/excavate.py
@@ -34,11 +34,38 @@ def find_subclasses(obj, base_class):
     >>> find_subclasses(locals(), A)
     [<class '__main__.B'>, <class '__main__.C'>]
     """
-    subclasses = []
-    for name, member in inspect.getmembers(obj):
-        if inspect.isclass(member) and issubclass(member, base_class) and member is not base_class:
-            subclasses.append(member)
-    return subclasses
+    subclasses = {}
+    for namespace in _member_namespaces(obj):
+        for name, member in namespace.items():
+            if inspect.isclass(member) and issubclass(member, base_class) and member is not base_class:
+                subclasses.setdefault(name, member)
+    return [subclasses[name] for name in sorted(subclasses)]
+
+
+def _snapshot(mapping):
+    # a live module's setup() may be assigning attributes while we read it
+    while True:
+        try:
+            return dict(mapping)
+        except RuntimeError:
+            continue
+
+
+def _member_namespaces(obj):
+    """Yield the namespaces of ``obj`` without invoking attribute descriptors.
+
+    ``inspect.getmembers`` getattrs every name, which fires properties such as
+    ``BaseModule.memory_usage``. That walks the module's ``__dict__`` and blows
+    up with "dictionary changed size during iteration" when the module it is
+    reading is concurrently running its own ``setup()``.
+    """
+    if not isinstance(obj, type):
+        instance_dict = getattr(obj, "__dict__", None)
+        if isinstance(instance_dict, dict):
+            yield _snapshot(instance_dict)
+    classes = obj.__mro__ if isinstance(obj, type) else type(obj).__mro__
+    for cls in classes:
+        yield _snapshot(vars(cls))
 
 
 def _pick_select_value(options_html):
diff --git a/bbot/test/test_step_2/module_tests/test_module_excavate.py b/bbot/test/test_step_2/module_tests/test_module_excavate.py
index 8f677ea56b..ee468d5105 100644
--- a/bbot/test/test_step_2/module_tests/test_module_excavate.py
+++ b/bbot/test/test_step_2/module_tests/test_module_excavate.py
@@ -6,6 +6,7 @@
 from bbot.modules.internal.excavate import ExcavateRule, split_yara_rules
 
 from pathlib import Path
+import threading
 import time
 import yara
 from bbot.test.worker import HTTPSERVER_PORT, HTTPSERVER_URL, LOCALHOST_URL
@@ -2186,3 +2187,52 @@ def check(self, module_test, events):
             "Both duplicate-content URLs were processed — content dedup failed for URL events"
         )
         assert len(consumer._content_dup_tracker) > 0, "Content dedup tracker should have entries"
+
+
+@pytest.mark.asyncio
+async def test_excavate_find_subclasses_does_not_touch_descriptors():
+    """excavate.setup() introspects sibling modules while their own setup() is
+    still assigning attributes. Reading through getattr fires BaseModule's
+    memory_usage property, which walks __dict__ and raises "dictionary changed
+    size during iteration" when it loses that race."""
+    from bbot.core.helpers.misc import get_size
+    from bbot.modules.internal.excavate import find_subclasses
+
+    class Sibling:
+        class SomeRule(ExcavateRule):
+            pass
+
+        def __init__(self):
+            for i in range(50):
+                setattr(self, f"attr{i}", i)
+
+        @property
+        def memory_usage(self):
+            return get_size(self, max_depth=3, seen=set())
+
+    sibling = Sibling()
+    assert [c.__name__ for c in find_subclasses(sibling, ExcavateRule)] == ["SomeRule"]
+
+    stop = []
+    errors = []
+
+    def concurrent_setup():
+        i = 0
+        while not stop:
+            setattr(sibling, f"cfg{i}", i)
+            delattr(sibling, f"cfg{i}")
+            i += 1
+
+    churn = threading.Thread(target=concurrent_setup, daemon=True)
+    churn.start()
+    try:
+        for _ in range(3000):
+            try:
+                assert find_subclasses(sibling, ExcavateRule) == [Sibling.SomeRule]
+            except RuntimeError as e:
+                errors.append(e)
+    finally:
+        stop.append(True)
+        churn.join(timeout=5)
+
+    assert not errors, f"find_subclasses raced against a concurrent setup(): {errors[0]}"

From 8f64c70c9a2c39d24585481691e924d33413cad9 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Mon, 31 Aug 2026 20:17:02 -0700
Subject: [PATCH 64/78] perf(deps): stop waiters blocking on a holder's
 unrelated installs

The lock-free precheck added in b9bdb4e25 only helps a worker that needs
nothing at all. A worker needing one uninstalled module still took the
exclusive install.lock, and the holder installs its entire module list
under a single hold, so the waiter paid the holder's whole chain.

Measured on CI run 33447872808 (py3.13, green): one worker ran a serial
ansible chain from 22:50:30 to 22:51:44 covering 18 modules (portscan
16s, retirejs 12s, gowitness 13s, medusa 18s). All four workers went
silent for that window and reported PASSED within 3s of each other. The
released waiter then logged "already done" for every module in 60ms; it
blocked 74s to do nothing. Across the run, three or more workers were
stalled simultaneously for 274s of 605s wall.

Two changes make the wait proportional to what the caller actually needs:

- install() polls with LOCK_EX|LOCK_NB instead of blocking, and after
  each observed publish rechecks _all_deps_satisfied. A waiter whose deps
  land mid-chain returns without ever taking the lock. Anything still
  unresolved falls through and installs under the lock as before.
- _install() publishes setup_status per module rather than once in the
  finally. Progress was previously invisible until the whole chain ended,
  so there was nothing for a waiter to observe.

The blocking flock was also called from async code, stalling the event
loop for the duration; polling with asyncio.sleep yields instead. This
is why the pre-fix regression test hangs rather than merely failing when
the holder is in-process.

_install_lock is now unused and removed.

Verified: waiter returns in 1.73s vs 6.51s blocked pre-fix, with the
holder as a real subprocess. Negative check on the old implementation
fails with "waiter blocked 6.51s". Unsatisfied waiters still take the
lock and install (asserted separately). test_modules_basic.py and
test_presets.py 38 passed. test_depsinstaller::test_depsinstaller fails
locally on the clean tree too (ansible/sudo), confirmed by stashing.
---
 bbot/core/helpers/depsinstaller/installer.py | 50 ++++++++++---
 bbot/test/test_step_1/test_depsinstaller.py  | 75 ++++++++++++++++++++
 2 files changed, 114 insertions(+), 11 deletions(-)

diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index 0d83702b37..abe6293b46 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -9,11 +9,12 @@
 import getpass
 import logging
 import uuid
+import asyncio
 from time import sleep
 from pathlib import Path
 from threading import Lock
 from itertools import chain
-from contextlib import contextmanager, suppress
+from contextlib import suppress
 from secrets import token_bytes
 from ansible_runner.interface import run
 from subprocess import CalledProcessError
@@ -27,6 +28,8 @@
 
 
 class DepsInstaller:
+    LOCK_POLL_INTERVAL = 0.25
+
     CORE_DEPS = {
         # core BBOT dependencies in the format of binary: package_name
         # each one will only be installed if the binary is not found
@@ -136,6 +139,7 @@ def __init__(self, parent_helper):
         self.data_dir = self.parent_helper.cache_dir / "depsinstaller"
         self.parent_helper.mkdir(self.data_dir)
         self.setup_status_cache = self.data_dir / "setup_status.json"
+        self._setup_status_stamp = None
         self.command_status = self.data_dir / "command_status"
         self.parent_helper.mkdir(self.command_status)
         self.ansible_artifact_dir = self.data_dir / "ansible_artifacts"
@@ -169,10 +173,25 @@ async def install(self, *modules):
         nothing_to_do = self._all_deps_satisfied(modules)
         if nothing_to_do is not None:
             return nothing_to_do
-        with self._install_lock():
+        # A holder installs every module it needs under a single lock hold, so blocking
+        # on the lock costs the holder's whole chain rather than just the part this
+        # caller needs. Poll instead, rechecking after each publish whether our own deps
+        # have arrived; a caller that needs nothing more then never takes the lock.
+        lock = open(self.data_dir / "install.lock", "w")
+        try:
+            while not self._try_lock(lock):
+                await asyncio.sleep(self.LOCK_POLL_INTERVAL)
+                if not self._setup_status_changed():
+                    continue
+                self.setup_status = self.read_setup_status()
+                satisfied = self._all_deps_satisfied(modules)
+                if satisfied is not None:
+                    return satisfied
             # another process may have installed deps while we waited for the lock
             self.setup_status = self.read_setup_status()
             return await self._install(*modules)
+        finally:
+            lock.close()
 
     def _all_deps_satisfied(self, modules):
         """Return (succeeded, failed) if every module is already installed, else None.
@@ -220,15 +239,21 @@ def _module_hash(self, preloaded):
             + str(__version__)
         ).hexdigest()
 
-    @contextmanager
-    def _install_lock(self):
-        lock_file = self.data_dir / "install.lock"
-        with open(lock_file, "w") as f:
-            fcntl.flock(f, fcntl.LOCK_EX)
-            try:
-                yield
-            finally:
-                fcntl.flock(f, fcntl.LOCK_UN)
+    def _try_lock(self, f):
+        try:
+            fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+        except OSError:
+            return False
+        return True
+
+    def _setup_status_changed(self):
+        try:
+            stamp = self.setup_status_cache.stat().st_mtime_ns
+        except OSError:
+            stamp = None
+        changed = stamp != self._setup_status_stamp
+        self._setup_status_stamp = stamp
+        return changed
 
     async def _install(self, *modules):
         await self.install_core_deps()
@@ -279,6 +304,9 @@ async def _install(self, *modules):
                         self.ensure_root(f'Module "{m}" needs root privileges to install its dependencies.')
                     success = await self.install_module(m)
                     self.setup_status[module_hash] = success
+                    # waiters poll this file to learn their own deps have landed, so
+                    # publish per module instead of once the whole chain is done
+                    self.write_setup_status()
                     if success or self.deps_behavior == "ignore_failed":
                         log.debug(f'Setup succeeded for module "{m}"')
                         succeeded.append(m)
diff --git a/bbot/test/test_step_1/test_depsinstaller.py b/bbot/test/test_step_1/test_depsinstaller.py
index 6d6ab8beee..fdd65858b6 100644
--- a/bbot/test/test_step_1/test_depsinstaller.py
+++ b/bbot/test/test_step_1/test_depsinstaller.py
@@ -1,3 +1,5 @@
+import time
+import subprocess
 from importlib.metadata import version as installed_version
 
 from ..bbot_fixtures import *
@@ -156,3 +158,76 @@ async def mock_install_core_deps():
         for module_name in (missing_module, present_module):
             preloaded.pop(module_name, None)
         await scan._cleanup()
+
+
+@pytest.mark.asyncio
+async def test_depsinstaller_waiter_does_not_block_on_unrelated_installs(monkeypatch, bbot_scanner):
+    """
+    A lock holder installs its whole module list under a single hold. A waiter whose own
+    deps land partway through that chain must return as soon as they land, rather than
+    blocking for the remainder of the holder's unrelated work.
+    """
+    scan = bbot_scanner("127.0.0.1")
+    await scan._prep()
+    installer = scan.helpers.depsinstaller
+
+    async def mock_install_core_deps():
+        return
+
+    status_cache = Path(scan.helpers.temp_filename())
+    monkeypatch.setattr(installer, "install_core_deps", mock_install_core_deps)
+    monkeypatch.setattr(installer, "setup_status_cache", status_cache)
+    monkeypatch.setattr(installer, "setup_status", {})
+    monkeypatch.setattr(installer, "_setup_status_stamp", None)
+    monkeypatch.setattr(installer, "_core_deps_cached", lambda: True)
+
+    mine = "deps_test_waiter_mine"
+    preloaded = scan.preset.module_loader._preloaded
+    preloaded[mine] = {
+        "sudo": False,
+        "deps": {
+            "apt": [],
+            "shell": [],
+            "pip": [],
+            "pip_constraints": [],
+            "common": [],
+            "ansible": [{"name": f"{mine} task"}],
+        },
+    }
+    mine_hash = installer._module_hash(preloaded[mine])
+
+    holder_chain_secs = 6
+    lock_path = str(installer.data_dir / "install.lock")
+    # a real second process, because the convoy this guards against is cross-process
+    holder_src = (
+        "import fcntl,json,sys,time\n"
+        "f=open(sys.argv[1],'w')\n"
+        "fcntl.flock(f,fcntl.LOCK_EX)\n"
+        "print('locked',flush=True)\n"
+        "time.sleep(0.5)\n"
+        "open(sys.argv[2],'w').write(json.dumps({sys.argv[3]:True}))\n"
+        f"time.sleep({holder_chain_secs})\n"
+    )
+    holder = subprocess.Popen(
+        [sys.executable, "-c", holder_src, lock_path, str(status_cache), mine_hash],
+        stdout=subprocess.PIPE,
+        text=True,
+    )
+    try:
+        assert holder.stdout.readline().strip() == "locked"
+
+        start = time.time()
+        succeeded, failed = await installer.install(mine)
+        waited = time.time() - start
+
+        assert not failed
+        assert succeeded == [mine]
+        assert waited < holder_chain_secs, (
+            f"waiter blocked {waited:.2f}s on the holder's unrelated work; "
+            f"it should have returned once its own dep was published"
+        )
+    finally:
+        holder.kill()
+        holder.wait()
+        preloaded.pop(mine, None)
+        await scan._cleanup()

From 96131a3b028330b1247f11031dfa283c729186a8 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 1 Sep 2026 08:36:50 -0700
Subject: [PATCH 65/78] perf(deps): skip per module pip calls already covered
 by the batch pass

_batch_pip_install() resolves every module's pip specs in one pass, then
install_module() immediately ran pip again for each module individually. Those
repeats install nothing: pip re-resolves, finds the package present, and exits.
Measured on a CI runner they are ~0.6s each of pure subprocess overhead, 22
calls in the py3.13 job, and they run inside the exclusive install lock so every
other xdist worker waits behind them.

install_module() now skips its pip call only for specs _batch_pip_install()
actually installed in this session, tracked in _batch_installed. Scoping it to
the batch's own specs rather than a general "is it satisfied" probe keeps
pip_install's --upgrade semantics intact: a spec the batch did not cover, a
stale cache entry whose package has since vanished, and force_install all still
take the normal install path.

Verified:
- cold path, 4 modules with uninstalled deps: 4 per module pip subprocesses
  before, 0 after, identical succeeded/failed sets
- negatively verified against the pre-fix tree, which shows the 4 calls
- force_install and unsatisfied specs still invoke pip
- test_depsinstaller.py matches the clean tree exactly (only the known
  environmental ansible/sudo failure)
- test_modules_basic.py + test_presets.py: 38 passed
---
 bbot/core/helpers/depsinstaller/installer.py | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index abe6293b46..a77493f3fe 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -162,6 +162,9 @@ def __init__(self, parent_helper):
 
         self.ensure_root_lock = Lock()
 
+        # pip specs already installed by _batch_pip_install() this session
+        self._batch_installed = set()
+
     async def install(self, *modules):
         # Concurrent scans (notably xdist workers) share the deps dir, so serialize
         # installs across processes: without this they race on the same files and
@@ -349,7 +352,13 @@ async def install_module(self, module):
         deps_pip = preloaded["deps"]["pip"]
         deps_pip_constraints = preloaded["deps"]["pip_constraints"]
         if deps_pip:
-            success &= await self.pip_install(deps_pip, constraints=deps_pip_constraints)
+            # _batch_pip_install() just installed these in one resolver pass, so repeating
+            # them per module spawns a no-op pip subprocess costing seconds. Only skip specs
+            # the batch actually covered, so --upgrade still runs for everything else.
+            if all(dep in self._batch_installed for dep in deps_pip):
+                log.debug(f'Pip dependencies for module "{module}" were installed in the batch pass')
+            else:
+                success &= await self.pip_install(deps_pip, constraints=deps_pip_constraints)
 
         # shared/common
         deps_common = preloaded["deps"]["common"]
@@ -634,7 +643,8 @@ async def _batch_pip_install(self, modules):
             return
 
         log.verbose(f"Batch-installing {len(packages):,} pip packages for {len(modules):,} modules")
-        await self.pip_install(packages)
+        if await self.pip_install(packages):
+            self._batch_installed.update(packages)
 
     def _pip_deps_satisfied(self, deps_pip):
         """Check whether a module's pip dependencies are currently installed in this environment.

From b143fb105f4c9c6f9eca0e99b09f2ca5be064354 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 1 Sep 2026 09:27:29 -0700
Subject: [PATCH 66/78] perf(deps): publish pip-only modules before the heavy
 builds

The holder installs its whole module set under one lock hold and publishes
setup_status per module, so a waiter is released only once its own modules
have been walked. _install() walked them in caller order, which interleaves
modules needing nothing but pip with ones that compile from source.

_batch_pip_install() has already satisfied every pip spec before the loop
starts, so a pip-only module is pure bookkeeping. Walking it behind medusa,
portscan, retirejs, gowitness and dnsbrute strands its waiters for the length
of those builds.

Measured on py3.13 run 33527062147: the batch pip pass completed at 15:41:29
but kafka's status was not published until 15:42:49, because kafka sat behind
the ansible chain in list order. aiokafka had been on disk for 80s. The kafka
test was credited 96.0s; it was blocked, not slow. Same for mongo, nats,
mysql, rabbitmq, neo4j, sqlite, postgres and web_report, all pip-only and all
published in the final 600ms of the chain.

Sort cheapest-first so every pip-only module publishes before any build
starts. The sort is a stable partition, so relative order within each group
is unchanged and the succeeded/failed sets are identical.

Verified: partition exact and stable over all 149 modules; the 15 modules
classified heavy match the 15 ansible spans in the CI log exactly; unknown
module names are preserved rather than dropped; with install_module stubbed,
every pip-only module now publishes before the first heavy build (kafka at
2.2% of chain wall time, previously last). Negatively verified by stashing:
pre-fix the first heavy build starts at index 5 while cheap modules keep
publishing through index 31.
---
 bbot/core/helpers/depsinstaller/installer.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index a77493f3fe..92f67a5e8f 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -258,6 +258,24 @@ def _setup_status_changed(self):
         self._setup_status_stamp = stamp
         return changed
 
+    def _install_order(self, modules):
+        """Order modules cheapest-first so waiters unblock as early as possible.
+
+        The holder installs its whole set under one lock hold and publishes
+        setup_status per module. A pip-only module is already satisfied by
+        _batch_pip_install(), so processing it behind a module that compiles from
+        source strands every waiter on it for the length of that build.
+        """
+
+        def cost(m):
+            preloaded = self.all_modules_preloaded.get(m)
+            if preloaded is None:
+                return 0
+            deps = preloaded["deps"]
+            return 1 if (deps["apt"] or deps["shell"] or deps["ansible"] or deps["common"]) else 0
+
+        return sorted(modules, key=cost)
+
     async def _install(self, *modules):
         await self.install_core_deps()
         succeeded = []
@@ -265,7 +283,7 @@ async def _install(self, *modules):
         await self._batch_pip_install(modules)
         try:
             notified = False
-            for m in modules:
+            for m in self._install_order(modules):
                 # assume success if we're ignoring dependencies
                 if self.deps_behavior == "disable":
                     succeeded.append(m)

From 15ddd0c697bc88f774d48c6ce7cb602e39ba9820 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 1 Sep 2026 11:56:25 -0700
Subject: [PATCH 67/78] perf(interactsh): skip the settle sleep when no canary
 was ever probed

dotnetnuke, lightfuzz, generic_ssrf and host_header all sleep 5s in finish()
before their final interactsh poll. The guard only checks that interactsh is
enabled and registered, which happens in setup() regardless of whether the
module ever sends a probe. A scan that loads one of these modules but produces
no events still pays the full 5s, and the scan loop cannot exit until finish()
returns.

Profiling `bbot -m dotnetnuke` showed 6.2s of a 10.6s run parked in epoll, with
a single 5.29s gap between "Completed finish()" and "Completed final finish()".
Sampling modules_finished confirmed dotnetnuke as the module holding the scan
open. This is one of 34 CLI invocations in test_cli_args and was the dominant
cost in that test on a warm runner.

Gate the sleep on the canary registry being non-empty. Each of these modules
resolves incoming interactions through that dict and returns early on a miss,
so an empty registry means every possible callback would be discarded anyway.
The dict is written at the probe sites themselves, so it is non-empty exactly
when a probe went out, which is exactly when the settle window has meaning.

`bbot -m dotnetnuke` 10.66s -> 5.44s. The host_header, generic_ssrf and
dotnetnuke module tests mock a real interaction and still pass, covering the
non-empty path.
---
 bbot/modules/dotnetnuke.py          | 2 +-
 bbot/modules/generic_ssrf.py        | 2 +-
 bbot/modules/host_header.py         | 2 +-
 bbot/modules/lightfuzz/lightfuzz.py | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/bbot/modules/dotnetnuke.py b/bbot/modules/dotnetnuke.py
index b99a7eeca0..784b0acfc0 100644
--- a/bbot/modules/dotnetnuke.py
+++ b/bbot/modules/dotnetnuke.py
@@ -210,7 +210,7 @@ async def cleanup(self):
                 self.warning(f"Interactsh failure: {e}")
 
     async def finish(self):
-        if self.interactsh_instance:
+        if self.interactsh_instance and self.interactsh_subdomain_tags:
             await self.helpers.sleep(5)
             try:
                 for r in await self.interactsh_instance.poll():
diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py
index ae0e1baab0..4fe7a1f44a 100644
--- a/bbot/modules/generic_ssrf.py
+++ b/bbot/modules/generic_ssrf.py
@@ -253,7 +253,7 @@ async def cleanup(self):
                 self.warning(f"Interactsh failure: {e}")
 
     async def finish(self):
-        if self.scan.config.get("interactsh_disable", False) is False:
+        if self.scan.config.get("interactsh_disable", False) is False and self.interactsh_subdomain_tags:
             await self.helpers.sleep(5)
             try:
                 for r in await self.interactsh_instance.poll():
diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py
index 02e9d60764..42e165a921 100644
--- a/bbot/modules/host_header.py
+++ b/bbot/modules/host_header.py
@@ -61,7 +61,7 @@ async def interactsh_callback(self, r):
                 self.debug("skipping results because subdomain tag was missing")
 
     async def finish(self):
-        if self.scan.config.get("interactsh_disable", False) is False:
+        if self.scan.config.get("interactsh_disable", False) is False and self.subdomain_tags:
             await self.helpers.sleep(5)
             try:
                 for r in await self.interactsh_instance.poll():
diff --git a/bbot/modules/lightfuzz/lightfuzz.py b/bbot/modules/lightfuzz/lightfuzz.py
index a2a3b71c5e..767a3c8690 100644
--- a/bbot/modules/lightfuzz/lightfuzz.py
+++ b/bbot/modules/lightfuzz/lightfuzz.py
@@ -351,7 +351,7 @@ async def cleanup(self):
                 self.warning(f"Interactsh failure: {e}")
 
     async def finish(self):
-        if self.interactsh_instance:
+        if self.interactsh_instance and self.interactsh_subdomain_tags:
             self.debug("finish(): sleeping 5s before final interactsh poll")
             await self.helpers.sleep(5)
             try:

From 7f5426dc073dbdd085ce372dee821641a6199c47 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 1 Sep 2026 13:07:05 -0700
Subject: [PATCH 68/78] perf(nuclei): cache budget-mode template analysis
 instead of reparsing 13.6k YAMLs

NucleiBudget.__init__ parsed the entire nuclei template tree on every budget-mode
scan setup. That is 13,619 YAML documents, and the work was pure recomputation:
the template set is identical from one scan to the next, so every scan paid ~9.5s
to derive the same budget_paths/collapsible_templates/severity_stats.

Three defects, all in product code rather than the test:

1. No caching. The analysis is a pure function of (template set, budget), but it
   ran unconditionally. Now keyed on a (path, size, mtime) signature over the
   template list plus the budget, and stored via the existing helpers cache. A
   template update changes the signature and forces a recompute; a different
   budget cannot collide with another budget's entry.

2. parse_yaml parsed templates that provably cannot contribute. Only templates
   with an http block ever yield a path through get_yaml_request_attr, but all
   13,619 were fed to the YAML loader. A byte-level check skips ~5.3k of them.
   Verified this loses nothing: parsing the excluded set yields 0 paths.

3. The parse cache was never released. _yaml_files retained every parsed document
   for the module's lifetime, ~343MB RSS, though it is dead once both passes
   finish. Cleared after construction.

Correctness verified ahead of speed. Compared against the previous implementation
run verbatim over budgets 1, 2, 5 and 10: budget_paths, collapsible_templates and
severity_stats are identical in all four. Cache invalidation checked positively
(touch a template, recompute fires, result still identical) and negatively
(budget is part of the key). Since cache_put is not atomic and the deps dir is
shared across xdist workers, a torn read matters: truncated, empty, garbage and
valid-json-missing-keys entries all fall back to recompute with identical output.

TestNucleiBudget setup 12.59s -> 2.70s warm; the file's 11 tests pass unchanged.
The win is per budget-mode scan, so real users doing repeated scans benefit too,
not just CI.
---
 bbot/modules/nuclei.py | 60 +++++++++++++++++++++++++++++++++++-------
 1 file changed, 51 insertions(+), 9 deletions(-)

diff --git a/bbot/modules/nuclei.py b/bbot/modules/nuclei.py
index ff314b7730..ee16f68390 100644
--- a/bbot/modules/nuclei.py
+++ b/bbot/modules/nuclei.py
@@ -9,6 +9,7 @@
 from contextlib import suppress
 
 from bbot.modules.base import BaseModule
+from bbot.core.helpers.misc import sha1
 from bbot.core.config.models import BaseModuleConfig, Field
 
 try:
@@ -468,12 +469,48 @@ def __init__(self, nuclei_module):
         self._yaml_files = {}
         self.templates_dir = nuclei_module.nuclei_templates_dir
         self.yaml_list = self.get_yaml_list()
-        self.budget_paths = self.find_budget_paths(nuclei_module.budget)
-        self.collapsible_templates, self.severity_stats = self.find_collapsible_templates()
+        cached = self._cache_load(nuclei_module.budget)
+        if cached is not None:
+            self.budget_paths, self.collapsible_templates, self.severity_stats = cached
+        else:
+            self.budget_paths = self.find_budget_paths(nuclei_module.budget)
+            self.collapsible_templates, self.severity_stats = self.find_collapsible_templates()
+            self._cache_store(nuclei_module.budget)
+        # the parsed documents are ~340MB and are dead once both passes are done
+        self._yaml_files = {}
 
     def get_yaml_list(self):
         return list(self.templates_dir.rglob("*.yaml"))
 
+    def _cache_key(self, budget):
+        """Identify the template set by (path, size, mtime) so a template update invalidates it."""
+        sig = []
+        for f in sorted(self.yaml_list):
+            st = f.stat()
+            sig.append((str(f), st.st_size, st.st_mtime_ns))
+        return f"nuclei_budget_{sha1(repr((budget, sig))).hexdigest()}"
+
+    def _cache_load(self, budget):
+        with suppress(Exception):
+            raw = self.parent.helpers.cache_get(self._cache_key(budget))
+            if raw:
+                d = json.loads(raw)
+                return d["budget_paths"], d["collapsible_templates"], d["severity_stats"]
+        return None
+
+    def _cache_store(self, budget):
+        with suppress(Exception):
+            self.parent.helpers.cache_put(
+                self._cache_key(budget),
+                json.dumps(
+                    {
+                        "budget_paths": self.budget_paths,
+                        "collapsible_templates": self.collapsible_templates,
+                        "severity_stats": self.severity_stats,
+                    }
+                ),
+            )
+
     # Given the current budget setting, scan all of the templates for paths, sort them by frequency and select the first N (budget) items
     def find_budget_paths(self, budget):
         path_frequency = {}
@@ -552,11 +589,16 @@ def find_collapsible_templates(self):
 
     def parse_yaml(self, yamlfile):
         if yamlfile not in self._yaml_files:
-            with open(yamlfile, "r") as stream:
-                try:
-                    y = yaml.load(stream, Loader=YamlLoader)
-                    self._yaml_files[yamlfile] = y
-                except yaml.YAMLError as e:
-                    self.parent.warning(f"failed to load yaml file: {e}")
-                    return {}
+            with open(yamlfile, "rb") as stream:
+                raw = stream.read()
+            # a template only contributes if it has an http block; skipping the rest
+            # avoids parsing ~5.3k of 13.6k templates that can never yield a path
+            if not (raw.startswith(b"http:") or b"\nhttp:" in raw):
+                self._yaml_files[yamlfile] = {}
+                return self._yaml_files[yamlfile]
+            try:
+                self._yaml_files[yamlfile] = yaml.load(raw, Loader=YamlLoader)
+            except yaml.YAMLError as e:
+                self.parent.warning(f"failed to load yaml file: {e}")
+                return {}
         return self._yaml_files[yamlfile]

From ebb820d96d69b0eec07a0f3c31ba19f0976b3c4a Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 1 Sep 2026 14:03:58 -0700
Subject: [PATCH 69/78] perf(test): drop nuclei manual-mode ratelimit floor and
 unshare its config dict

Two independent defects in test_module_nuclei.py, one perf and one correctness.

Perf: TestNucleiManual set "ratelimit": 10 while its template dir
(http/miscellaneous/) issues 100 requests against the fixture server. nuclei
paces those at 10/s, so every batch had a hard 10s floor, and the module runs
two batches per test. Measured with the fixture server counting requests:
at rl=10 the 100 requests span 9.88s of the 10.26s run, so the wall time IS
the pacing. The subclasses that inherit this config (CustomHeaders,
EnvIsolation) paid it too.

The limit is not load-bearing. The assertions check WHICH findings nuclei
reports (dir-listing, old-copyright), not request pacing, and the result set
is identical across rate limits: 5 runs each at rl=10, rl=150/conc=2 and
rl=150/conc=25 produced exactly one distinct result set,
{dir-listing, old-copyright}, with no flake. Dropping the override falls back
to the module default (150) and keeps every assertion intact.

Correctness: TestNucleiCustomHeaders did
`config_overrides = TestNucleiManual.config_overrides` and then wrote
`config_overrides["web"]["http_headers"]` into it. That is the same dict
object, so defining the subclass mutated the parent at import time. Verified:
TestNucleiManual.config_overrides["web"] and TestNucleiEnvIsolation's both
carried CustomHeaders' testheader1/testheader2, and
`Manual.config_overrides is CustomHeaders.config_overrides` was True. Both
tests were silently running with custom HTTP headers they never asked for,
and the leak was order dependent on class-definition order. Now deep copied;
a shallow dict() would not fix it because the mutated "web" dict is nested.

Verified:
- Assertions still discriminate, proven by mutation: pointing http_headers at
  {"wrong": "value"} makes TestNucleiCustomHeaders fail on `assert
  first_run_detect`, and it passes again when restored.
- Isolation holds: Manual/EnvIsolation "web" is now
  {spider_distance, spider_depth} with no http_headers; CustomHeaders keeps
  its own.
- Whole file 11 passed, 3x before (59.7s/75.2s/66.0s, median 66.0s) vs 3x
  after (68.0s/48.2s/58.7s, median 58.7s). Per-test setup is the cleaner
  signal: TestNucleiCustomHeaders 27.60s -> 10.58s.
- ruff check and ruff format --check clean.
---
 bbot/test/test_step_2/module_tests/test_module_nuclei.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_2/module_tests/test_module_nuclei.py b/bbot/test/test_step_2/module_tests/test_module_nuclei.py
index 5885787c91..0f8790e924 100644
--- a/bbot/test/test_step_2/module_tests/test_module_nuclei.py
+++ b/bbot/test/test_step_2/module_tests/test_module_nuclei.py
@@ -2,6 +2,7 @@
 from .base import ModuleTestBase
 from bbot.test.worker import HTTPSERVER_URL, BBOT_TEST_DIR, BBOT_TEST_TOOLS_DIR
 
+import copy
 import fcntl
 from types import SimpleNamespace
 from unittest.mock import patch
@@ -20,7 +21,6 @@ class TestNucleiManual(ModuleTestBase):
             "nuclei": {
                 "mode": "manual",
                 "concurrency": 2,
-                "ratelimit": 10,
                 "templates": f"{BBOT_TEST_TOOLS_DIR}/nuclei-state/templates/http/miscellaneous/",
                 "directory_only": False,
             }
@@ -387,7 +387,9 @@ async def failing_update():
 
 class TestNucleiCustomHeaders(TestNucleiManual):
     custom_headers = {"testheader1": "test1", "testheader2": "test2"}
-    config_overrides = TestNucleiManual.config_overrides
+    # deep copy: a shallow dict() still shares the nested "web" dict, so writing
+    # http_headers into it would reach TestNucleiManual and every sibling subclass.
+    config_overrides = copy.deepcopy(TestNucleiManual.config_overrides)
     config_overrides["web"]["http_headers"] = custom_headers
 
     async def setup_after_prep(self, module_test):

From 068cea47ee483482eb8208df19eb99aaf934eb72 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 1 Sep 2026 14:41:09 -0700
Subject: [PATCH 70/78] perf(test): build the e2e venv with uv so it reuses the
 warm wheel cache

The bbot_venv fixture built a stdlib venv and ran `pip install -e .` into it,
resolving and installing ~84 packages from scratch every time. Profiled locally:
venv create 4.01s, pip install 25.32s. Nothing in that work is cold; CI already
runs `uv sync --group dev` in the same job, so uv's shared wheel cache is warm
by the time this fixture runs, but pip has its own cache and cannot see it.

Switch to `uv venv` + `uv pip install -e`, falling back to the stdlib venv and
pip when uv is not on PATH, so the fixture still works outside CI.

Package sets are identical, not merely similar: compared `pip list` from both
paths, 67 packages each, zero entries unique to either side and zero version
differences.

Verified:
- uv path 0.89s vs pip path 25.66s; both produce bin/bbot and `bbot -h` exits 0.
- pip fallback exercised directly, not just by inspection.
- Same file, same 4 tests pass on both: setup 24.62s -> 2.70s,
  file wall 39.50s -> 19.43s.
- ruff check + ruff format --check clean.

No assertions changed. The existing bbot CLI assert still guards the install.
---
 bbot/test/test_step_1/test_e2e.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/bbot/test/test_step_1/test_e2e.py b/bbot/test/test_step_1/test_e2e.py
index 25745f1029..aee0831153 100644
--- a/bbot/test/test_step_1/test_e2e.py
+++ b/bbot/test/test_step_1/test_e2e.py
@@ -8,6 +8,7 @@
 """
 
 import json
+import shutil
 import subprocess
 import sys
 import os
@@ -32,15 +33,22 @@ def _find_repo_root():
 
 @pytest.fixture(scope="module")
 def bbot_venv(tmp_path_factory):
-    """Create a fresh virtualenv and pip-install bbot from the local checkout."""
+    """Create a fresh virtualenv and install bbot from the local checkout."""
     venv_dir = tmp_path_factory.mktemp("bbot_e2e_venv")
     repo_root = _find_repo_root()
 
-    subprocess.check_call([sys.executable, "-m", "venv", str(venv_dir)])
-    pip = str(venv_dir / "bin" / "pip")
-    bbot = str(venv_dir / "bin" / "bbot")
+    # uv reuses its shared wheel cache, which CI already warmed via `uv sync`
+    uv = shutil.which("uv")
+    if uv:
+        subprocess.check_call([uv, "venv", str(venv_dir), "--python", sys.executable, "--seed", "--quiet"])
+        venv_python = str(venv_dir / "bin" / "python")
+        subprocess.check_call([uv, "pip", "install", "-e", repo_root, "--python", venv_python, "--quiet"], timeout=300)
+    else:
+        subprocess.check_call([sys.executable, "-m", "venv", str(venv_dir)])
+        pip = str(venv_dir / "bin" / "pip")
+        subprocess.check_call([pip, "install", "-e", repo_root, "--quiet"], timeout=300)
 
-    subprocess.check_call([pip, "install", "-e", repo_root, "--quiet"], timeout=300)
+    bbot = str(venv_dir / "bin" / "bbot")
     assert os.path.isfile(bbot), f"bbot CLI not found at {bbot}"
     return bbot
 

From c26fa86aa99d56aa5547a177fdde5637c949d616 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 1 Sep 2026 15:41:13 -0700
Subject: [PATCH 71/78] perf(helpers): stop re-splatting kwargs at every node
 in search_format_dict

search_format_dict recursed as `search_format_dict(v, **kwargs)`. Every
recursive call re-splatted the mapping into a fresh dict, so the walk cost
O(nodes * len(kwargs)) instead of O(nodes). The substitution itself is trivial;
the cost was entirely in rebuilding the kwargs dict 6.6k times per call.

This is on the Scanner construction path. Preset.bake() calls
ModuleLoader.find_and_replace(**os_environ), which runs search_format_dict over
the full preloaded module set. Measured on the real payload: 6632 nodes, 3190
strings, and only 43 strings that contain a placeholder at all, against 394
environment keys. So ~394 dict rebuilds per node to service 43 actual
substitutions.

Confirmed the scaling is in the kwargs width, not the tree:
  kwargs=  3 -> 0.0051s
  kwargs= 50 -> 0.0268s
  kwargs=200 -> 0.0715s
  kwargs=394 -> 0.1476s

Fix keeps the public **kwargs signature (test_helpers and the docstring example
depend on it) and moves the recursion into a helper that passes the mapping by
reference.

Measured:
  search_format_dict(preloaded)     0.1547s -> 0.0027s  (57x)
  search_format_dict(_shared_deps)  0.0054s -> 0.0001s  (40x)
  find_and_replace                  0.1593s -> 0.0032s
  Scanner.__init__                  0.1633s -> 0.0146s  (11x)

Every module test builds a Scanner, so this is a per-test floor across the whole
suite, not a single-file win. test_module_lightfuzz.py (114 tests): 332.1s ->
310.8s, same 114 passed, median per-test delta 0.218s, which matches the
measured per-construction cost.

Equivalence proved against the original implementation, not assumed: 23/23 edge
cases identical (missing keys, empty/non-str/non-dict inputs, non-string dict
keys, malformed and nested placeholders, bytes/set/tuple values), plus exact
output equality on the real preloaded and _shared_deps payloads.

test_helpers.py, test_modules_basic.py, test_presets.py, test_scan.py,
test_python_api.py, test_depsinstaller.py, test_config.py, test_events.py all
pass.
---
 bbot/core/helpers/misc.py | 24 +++++++++++++++---------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/bbot/core/helpers/misc.py b/bbot/core/helpers/misc.py
index 969df0f287..28b65ea9e9 100644
--- a/bbot/core/helpers/misc.py
+++ b/bbot/core/helpers/misc.py
@@ -1444,6 +1444,20 @@ def search_dict_by_key(key, d):
 _PLACEHOLDER_REGEX = re.compile(r"#\{(\w+)\}")
 
 
+def _search_format_dict(d, replacements):
+    # replacements travels by reference; re-splatting it at each node rebuilds
+    # the whole mapping once per node, making the walk O(nodes * replacements).
+    if isinstance(d, dict):
+        return {k: _search_format_dict(v, replacements) for k, v in d.items()}
+    elif isinstance(d, list):
+        return [_search_format_dict(v, replacements) for v in d]
+    elif isinstance(d, str):
+        if "#{" not in d:
+            return d
+        return _PLACEHOLDER_REGEX.sub(lambda m: replacements.get(m.group(1), m.group(0)), d)
+    return d
+
+
 def search_format_dict(d, **kwargs):
     """Recursively format string values in a dictionary or list using the provided keyword arguments.
 
@@ -1458,15 +1472,7 @@ def search_format_dict(d, **kwargs):
         >>> search_format_dict({"test": "#{name} is awesome"}, name="keanu")
         {"test": "keanu is awesome"}
     """
-    if isinstance(d, dict):
-        return {k: search_format_dict(v, **kwargs) for k, v in d.items()}
-    elif isinstance(d, list):
-        return [search_format_dict(v, **kwargs) for v in d]
-    elif isinstance(d, str):
-        if "#{" not in d:
-            return d
-        return _PLACEHOLDER_REGEX.sub(lambda m: kwargs.get(m.group(1), m.group(0)), d)
-    return d
+    return _search_format_dict(d, kwargs)
 
 
 def search_dict_values(d, *regexes):

From edc01ac9a596279b4d5793ab346bfd948e4aa049 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 1 Sep 2026 17:06:48 -0700
Subject: [PATCH 72/78] perf(deps): batch already-satisfied pip specs instead
 of dropping them

_batch_pip_install() skipped any spec already satisfied in the environment,
so those specs never entered _batch_installed. install_module() keys its skip
off that set, so every module whose deps were already present fell through and
spawned its own no-op `pip install --upgrade` subprocess.

On a warm CI venv all 18 batchable specs are already satisfied, so the batch
pass installed nothing and 20 modules each paid a full pip startup. Measured on
the --install-all-deps path: 20 subprocesses / 19.2s -> 1 / 2.1s. That call is
the bulk of test_cli_args, the slowest test in the suite at 140.2s.

Satisfied specs are now batched rather than dropped, so `--upgrade` still runs
for them, in one resolver pass instead of one subprocess per module. Two guards
keep the covered set honest:

- _needs_install() gates which modules contribute specs, mirroring the branch
  in _install(). Without it the batch would install for modules the locked path
  would never have touched. Verified to agree with _install() on all 915
  (module, deps_behavior, cache-state) combinations.
- a spec shared with a module carrying custom pip_constraints is excluded from
  the covered set, since it must still be resolved against those constraints.

_batch_installed is now reset per pass; it was instance-scoped and never
cleared, so a second install() in the same process saw stale coverage and could
suppress a genuinely needed install.

A failed batch covers nothing, so every module still falls back to its own
install. Verified.

test_depsinstaller_stale_pip_cache asserted the per-subprocess call shape, which
this legitimately changes; it now asserts the set of specs that reached pip,
which is the property it was actually protecting. Added
test_depsinstaller_batch_covers_satisfied_deps, confirmed to fail against the
old implementation with the exact per-module split and pass against the new.
---
 bbot/core/helpers/depsinstaller/installer.py | 40 ++++++---
 bbot/test/test_step_1/test_depsinstaller.py  | 91 +++++++++++++++++++-
 2 files changed, 118 insertions(+), 13 deletions(-)

diff --git a/bbot/core/helpers/depsinstaller/installer.py b/bbot/core/helpers/depsinstaller/installer.py
index 92f67a5e8f..7a89ecfb77 100644
--- a/bbot/core/helpers/depsinstaller/installer.py
+++ b/bbot/core/helpers/depsinstaller/installer.py
@@ -162,7 +162,7 @@ def __init__(self, parent_helper):
 
         self.ensure_root_lock = Lock()
 
-        # pip specs already installed by _batch_pip_install() this session
+        # pip specs covered by the current batch pass; install_module() skips these
         self._batch_installed = set()
 
     async def install(self, *modules):
@@ -630,31 +630,35 @@ def _core_dep_satisfied(self, command):
     async def _batch_pip_install(self, modules):
         """Pre-install every module's pip deps in one resolver pass.
 
-        install_module() still runs per module afterward; by then the packages are
-        already present, so its own pip call is a no-op. Only modules using the
-        default constraints are batched, since custom constraints must be resolved
-        against their own set.
+        install_module() still runs per module afterward, but skips any spec this pass
+        covered. Already-satisfied specs are batched rather than dropped so that
+        --upgrade still runs for them, in one resolver pass instead of one subprocess
+        per module. Only modules using the default constraints are batched, since
+        custom constraints must be resolved against their own set.
         """
+        # scoped to this pass: a later install() call may face a different environment,
+        # so stale coverage must never suppress its per-module installs
+        self._batch_installed = set()
         if self.deps_behavior == "disable":
             return
 
         packages = []
         seen = set()
+        constrained = set()
         for m in modules:
             preloaded = self.all_modules_preloaded.get(m)
             if not preloaded:
                 continue
+            if not self._needs_install(preloaded):
+                continue
             deps = preloaded.get("deps", {})
             if deps.get("pip_constraints"):
+                constrained.update(deps.get("pip", []))
                 continue
             for dep in deps.get("pip", []):
                 if dep in seen:
                     continue
                 seen.add(dep)
-                if self.deps_behavior != "force_install":
-                    satisfied, _ = self._pip_deps_satisfied([dep])
-                    if satisfied:
-                        continue
                 packages.append(dep)
 
         if len(packages) < 2:
@@ -662,7 +666,23 @@ async def _batch_pip_install(self, modules):
 
         log.verbose(f"Batch-installing {len(packages):,} pip packages for {len(modules):,} modules")
         if await self.pip_install(packages):
-            self._batch_installed.update(packages)
+            # a spec shared with a module carrying custom constraints must still be
+            # resolved against those constraints, so it never counts as covered
+            self._batch_installed.update(set(packages) - constrained)
+
+    def _needs_install(self, preloaded):
+        """Whether _install() will actually reach install_module() for this module.
+
+        Batching deps for a module that is already recorded done would install
+        packages the locked path would never have touched.
+        """
+        if self.deps_behavior in ("disable", "force_install"):
+            return self.deps_behavior == "force_install"
+        success = self.setup_status.get(self._module_hash(preloaded), None)
+        if success is True:
+            satisfied, _ = self._pip_deps_satisfied(preloaded["deps"]["pip"])
+            return not satisfied
+        return success is None or self.deps_behavior == "retry_failed"
 
     def _pip_deps_satisfied(self, deps_pip):
         """Check whether a module's pip dependencies are currently installed in this environment.
diff --git a/bbot/test/test_step_1/test_depsinstaller.py b/bbot/test/test_step_1/test_depsinstaller.py
index fdd65858b6..363525c1db 100644
--- a/bbot/test/test_step_1/test_depsinstaller.py
+++ b/bbot/test/test_step_1/test_depsinstaller.py
@@ -1,5 +1,6 @@
 import time
 import subprocess
+from itertools import chain
 from importlib.metadata import version as installed_version
 
 from ..bbot_fixtures import *
@@ -141,11 +142,13 @@ async def mock_install_core_deps():
         }
 
     try:
-        # first run: nothing is cached yet, so both modules install their deps
+        # first run: nothing is cached yet, so both modules install their deps.
+        # the batch pass covers them in one resolver call, so assert on the set of
+        # specs that reached pip rather than on how many subprocesses carried them
         succeeded, failed = await installer.install(missing_module, present_module)
         assert not failed
         assert succeeded == sorted([missing_module, present_module])
-        assert sorted(pip_installs) == [["bbot-nonexistent-package"], ["pydantic"]]
+        assert sorted(chain(*pip_installs)) == ["bbot-nonexistent-package", "pydantic"]
 
         # second run: the cache is warm, but only pydantic is actually installed.
         # the missing package must be reinstalled instead of silently assumed present
@@ -153,13 +156,95 @@ async def mock_install_core_deps():
         succeeded, failed = await installer.install(missing_module, present_module)
         assert not failed
         assert succeeded == sorted([missing_module, present_module])
-        assert pip_installs == [["bbot-nonexistent-package"]]
+        assert sorted(chain(*pip_installs)) == ["bbot-nonexistent-package"]
     finally:
         for module_name in (missing_module, present_module):
             preloaded.pop(module_name, None)
         await scan._cleanup()
 
 
+@pytest.mark.asyncio
+async def test_depsinstaller_batch_covers_satisfied_deps(monkeypatch, bbot_scanner):
+    """
+    The batch pass must cover every spec it installs, including ones already present in
+    the environment. Leaving those uncovered made each owning module spawn its own no-op
+    pip subprocess, which dominated `bbot --install-all-deps`.
+    """
+    scan = bbot_scanner("127.0.0.1")
+    await scan._prep()
+    installer = scan.helpers.depsinstaller
+
+    pip_installs = []
+
+    async def mock_pip_install(packages, constraints=None):
+        pip_installs.append(list(packages))
+        return True
+
+    async def mock_install_core_deps():
+        return
+
+    monkeypatch.setattr(installer, "pip_install", mock_pip_install)
+    monkeypatch.setattr(installer, "install_core_deps", mock_install_core_deps)
+    monkeypatch.setattr(installer, "setup_status_cache", scan.helpers.temp_filename())
+    monkeypatch.setattr(installer, "setup_status", {})
+
+    preloaded = scan.preset.module_loader._preloaded
+    # pydantic is installed in every test environment, so these are the "satisfied" case
+    # distinct specs so the two modules hash differently; both are already installed,
+    # which is precisely the case the batch pass used to leave uncovered
+    shared_modules = ["deps_batch_a", "deps_batch_b"]
+    shared_specs = {"deps_batch_a": "pydantic", "deps_batch_b": "orjson"}
+    constrained_module = "deps_batch_constrained"
+    for module_name in shared_modules:
+        preloaded[module_name] = {
+            "sudo": False,
+            "deps": {
+                "apt": [],
+                "shell": [],
+                "pip": [shared_specs[module_name]],
+                "pip_constraints": [],
+                "common": [],
+                "ansible": [],
+            },
+        }
+    preloaded[constrained_module] = {
+        "sudo": False,
+        "deps": {
+            "apt": [],
+            "shell": [],
+            "pip": ["pydantic"],
+            "pip_constraints": ["pydantic>=1.0"],
+            "common": [],
+            "ansible": [],
+        },
+    }
+
+    try:
+        succeeded, failed = await installer.install(*shared_modules)
+        assert not failed
+        assert succeeded == sorted(shared_modules)
+        # one resolver pass covers both modules; neither may add a subprocess of its own
+        assert pip_installs == [["pydantic", "orjson"]]
+
+        # a spec shared with a module carrying custom constraints must still be resolved
+        # against those constraints, so the batch may not claim it
+        pip_installs.clear()
+        # install() re-reads the status cache under the lock, so a fresh file is what
+        # actually makes this a cold run
+        monkeypatch.setattr(installer, "setup_status_cache", scan.helpers.temp_filename())
+        monkeypatch.setattr(installer, "setup_status", {})
+        succeeded, failed = await installer.install(*shared_modules, constrained_module)
+        assert not failed
+        assert succeeded == sorted([*shared_modules, constrained_module])
+        # pydantic is claimed by the constrained module, so only orjson may be covered
+        assert installer._batch_installed == {"orjson"}
+        assert ["pydantic"] in pip_installs
+    finally:
+        for module_name in (*shared_modules, constrained_module):
+            preloaded.pop(module_name, None)
+        await scan._cleanup()
+
+
 @pytest.mark.asyncio
 async def test_depsinstaller_waiter_does_not_block_on_unrelated_installs(monkeypatch, bbot_scanner):
     """

From 3428b7e3d4b35a4e4c4e4b9d9542a912db0864d4 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 1 Sep 2026 18:18:10 -0700
Subject: [PATCH 73/78] perf(test): stop module-loading tests from queueing on
 the deps install lock

test_module_loading and test_modules_basic_perdomainonly each request the full
module set, so load_modules() calls depsinstaller.install() for all 149 modules.
Neither test's body needs a single dependency on disk: both only instantiate
modules and read class attributes off them.

Under xdist that request is not free. install() takes a lock-free fast path only
when every module is already recorded installed, and while another worker holds
install.lock mid-install that condition is false, so these two fall through to
the polling wait. The holder installs its whole chain under one hold, so a
waiter needing the complete module set cannot be released early and pays the
holder's entire remaining runtime.

That is the whole cost of both tests. Measured against a foreign process holding
install.lock for 12s, with the fast path declining as it does in CI:

  deps default   wall 12.03s   49 lock poll attempts
  deps disable   wall  0.00s    1 attempt, never blocks

CI shows the same shape: py3.13 job 100073737907 credits test_module_loading
59.3s and perdomainonly 53.3s, and both release within seconds of gw2 finishing
test_cli_args --install-all-deps. Their real bodies are ~1.6s and ~0.9s.

Setting deps behavior to disable makes install() return immediately without
touching the lock. Verified it does not weaken either test: both configurations
load an identical set of 151 modules with identical _type and watched_events,
and the same four per_domain_only modules (azure_tenant, emailformat, skymem,
viewdns). Mutation check confirms the async-hook assertion still fires under
disable: patching nuclei.handle_event and wayback.cleanup to non-async is caught
with both names reported.

No assertions changed, no coverage reduced. Full file: 7 passed in 4.00s.
---
 bbot/test/test_step_1/test_modules_basic.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 2d34ed41eb..37ecb30ded 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -338,10 +338,15 @@ class mod_domain_only(BaseModule):
 
 @pytest.mark.asyncio
 async def test_modules_basic_perdomainonly(bbot_scanner, monkeypatch):
+    config = {i: True for i in available_internal_modules if i != "dnsresolve"}
+    # loading reads class attributes only, so no tool needs to be on disk. Leaving
+    # deps enabled makes this block on the install lock for the whole of whichever
+    # worker is installing, which is the entire cost of this test under xdist.
+    config["deps"] = {"behavior": "disable"}
     per_domain_scan = bbot_scanner(
         "evilcorp.com",
         modules=list(available_modules),
-        config={i: True for i in available_internal_modules if i != "dnsresolve"},
+        config=config,
         force_start=True,
     )
 
@@ -531,10 +536,14 @@ async def handle_event(self, event):
 
 @pytest.mark.asyncio
 async def test_module_loading(bbot_scanner):
+    config = {i: True for i in available_internal_modules if i != "dnsresolve"}
+    # same reason as perdomainonly: this only reads class attributes, so waiting on
+    # the shared install lock is pure dead time under xdist.
+    config["deps"] = {"behavior": "disable"}
     scan2 = bbot_scanner(
         modules=list(available_modules),
         output_modules=list(available_output_modules),
-        config={i: True for i in available_internal_modules if i != "dnsresolve"},
+        config=config,
         force_start=True,
     )
     # every assertion below reads class attributes off the instantiated modules, so

From 3dbcab1f1ba046a87a48f94538be90c6128aee65 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Tue, 1 Sep 2026 18:29:08 -0700
Subject: [PATCH 74/78] Revert "perf(test): stop module-loading tests from
 queueing on the deps install lock"

This reverts commit 3428b7e3d4b35a4e4c4e4b9d9542a912db0864d4.

Broke test_module_loading and test_modules_basic_perdomainonly on all five
python versions:

  bbot.errors.BBOTError: Error loading module badsecrets:
  No module named 'badsecrets'

The premise was wrong. I assumed loading a module only reads class attributes,
so no dependency needs to be on disk. But _load_modules() imports each module's
python file, and a module whose import pulls a third-party package needs that
package actually installed. badsecrets declares deps_pip = ["badsecrets~=1.2.1"]
and is NOT in pyproject.toml, runtime or dev, so it reaches the venv only via
the deps installer. Setting deps behavior to disable skips that install, and the
import then fails.

The local equivalence check that cleared this was invalid: my venv already had
badsecrets from earlier runs, so both configurations loaded 151 modules and the
difference was invisible. Comparing deps-on against deps-off in an environment
already carrying the deps proves nothing. A valid check has to run against a
venv holding only pyproject dependencies.

The convoy measurement itself still stands (a waiter needing the full module set
tracks the holder exactly, 12.03s vs 0.00s under a held lock). Only this way of
avoiding it is wrong. Any real fix has to keep the deps installed and instead
stop the waiter from serializing behind the holder's entire chain.
---
 bbot/test/test_step_1/test_modules_basic.py | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/bbot/test/test_step_1/test_modules_basic.py b/bbot/test/test_step_1/test_modules_basic.py
index 37ecb30ded..2d34ed41eb 100644
--- a/bbot/test/test_step_1/test_modules_basic.py
+++ b/bbot/test/test_step_1/test_modules_basic.py
@@ -338,15 +338,10 @@ class mod_domain_only(BaseModule):
 
 @pytest.mark.asyncio
 async def test_modules_basic_perdomainonly(bbot_scanner, monkeypatch):
-    config = {i: True for i in available_internal_modules if i != "dnsresolve"}
-    # loading reads class attributes only, so no tool needs to be on disk. Leaving
-    # deps enabled makes this block on the install lock for the whole of whichever
-    # worker is installing, which is the entire cost of this test under xdist.
-    config["deps"] = {"behavior": "disable"}
     per_domain_scan = bbot_scanner(
         "evilcorp.com",
         modules=list(available_modules),
-        config=config,
+        config={i: True for i in available_internal_modules if i != "dnsresolve"},
         force_start=True,
     )
 
@@ -536,14 +531,10 @@ async def handle_event(self, event):
 
 @pytest.mark.asyncio
 async def test_module_loading(bbot_scanner):
-    config = {i: True for i in available_internal_modules if i != "dnsresolve"}
-    # same reason as perdomainonly: this only reads class attributes, so waiting on
-    # the shared install lock is pure dead time under xdist.
-    config["deps"] = {"behavior": "disable"}
     scan2 = bbot_scanner(
         modules=list(available_modules),
         output_modules=list(available_output_modules),
-        config=config,
+        config={i: True for i in available_internal_modules if i != "dnsresolve"},
         force_start=True,
     )
     # every assertion below reads class attributes off the instantiated modules, so

From 76398d67802db805c97fd420d06a2c5cf69cd17a Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Wed, 2 Sep 2026 08:03:49 -0700
Subject: [PATCH 75/78] perf(interactsh): make the final-poll settle wait a
 property of the transport

Four modules hardcoded `await self.helpers.sleep(5)` in `finish()` before
their last interactsh poll. That wait exists to let interactions triggered
just before the scan ended propagate to the interact.sh server, so it is a
property of the transport, not of each module.

Under test the transport is `Interactsh_mock`, an in-process asyncio.Queue
with no propagation delay at all, so every interactsh test paid the full 5s
for interactions that were already queued. `scan.finish()` re-queues FINISHED
whenever the previous round produced new activity, so `finish()` runs at
least twice per scan and the cost is 10s per test, not 5s. Traced it: the
second round's poll returns 0 interactions every time.

Adds `Interactsh.settle()`, which keeps the identical 5s `asyncio.sleep` for
the real client, and overrides it to a no-op on the mock. Production timing
is unchanged.

Verified: interactsh subset of test_module_lightfuzz.py 43.4s -> 19.2s.
generic_ssrf + host_header + dotnetnuke 52s -> 21s, same pass set and same
finding counts (60 and 30), so no interactions are lost.

Note on what was deliberately NOT changed: the mock's own sleeps (0.5s
poll_loop idle, 0.1s per interaction, 1s in deregister) are load-bearing.
Removing them individually passes, but removing them together drops 4 of 60
findings in generic_ssrf. That is a real ordering dependency in the mock's
drain path, not a stale assertion, so the sleeps stay until it is understood.
---
 bbot/core/helpers/interactsh.py     | 11 +++++++++++
 bbot/modules/dotnetnuke.py          |  2 +-
 bbot/modules/generic_ssrf.py        |  2 +-
 bbot/modules/host_header.py         |  2 +-
 bbot/modules/lightfuzz/lightfuzz.py |  4 ++--
 bbot/test/conftest.py               |  4 ++++
 6 files changed, 20 insertions(+), 5 deletions(-)

diff --git a/bbot/core/helpers/interactsh.py b/bbot/core/helpers/interactsh.py
index e178c6db8a..0d892c5129 100644
--- a/bbot/core/helpers/interactsh.py
+++ b/bbot/core/helpers/interactsh.py
@@ -78,6 +78,9 @@ class Interactsh:
         ```
     """
 
+    # grace period for in-flight interactions to reach the server before a final poll
+    SETTLE_INTERVAL = 5
+
     def __init__(self, parent_helper, poll_interval=10):
         self.parent_helper = parent_helper
         self.server = None
@@ -87,6 +90,14 @@ def __init__(self, parent_helper, poll_interval=10):
         self.poll_interval = poll_interval
         self._poll_task = None
 
+    async def settle(self):
+        """Wait for interactions triggered just before the scan ended to reach the server.
+
+        Round-trip latency is a property of the transport, so mocks override this to zero
+        rather than every caller hardcoding a sleep.
+        """
+        await asyncio.sleep(self.SETTLE_INTERVAL)
+
     async def register(self, callback=None):
         """
         Registers the instance with an interact.sh server and sets up polling.
diff --git a/bbot/modules/dotnetnuke.py b/bbot/modules/dotnetnuke.py
index 784b0acfc0..8d754f84d3 100644
--- a/bbot/modules/dotnetnuke.py
+++ b/bbot/modules/dotnetnuke.py
@@ -211,7 +211,7 @@ async def cleanup(self):
 
     async def finish(self):
         if self.interactsh_instance and self.interactsh_subdomain_tags:
-            await self.helpers.sleep(5)
+            await self.interactsh_instance.settle()
             try:
                 for r in await self.interactsh_instance.poll():
                     await self.interactsh_callback(r)
diff --git a/bbot/modules/generic_ssrf.py b/bbot/modules/generic_ssrf.py
index 4fe7a1f44a..0d040f14e5 100644
--- a/bbot/modules/generic_ssrf.py
+++ b/bbot/modules/generic_ssrf.py
@@ -254,7 +254,7 @@ async def cleanup(self):
 
     async def finish(self):
         if self.scan.config.get("interactsh_disable", False) is False and self.interactsh_subdomain_tags:
-            await self.helpers.sleep(5)
+            await self.interactsh_instance.settle()
             try:
                 for r in await self.interactsh_instance.poll():
                     await self.interactsh_callback(r)
diff --git a/bbot/modules/host_header.py b/bbot/modules/host_header.py
index 42e165a921..237e61cb21 100644
--- a/bbot/modules/host_header.py
+++ b/bbot/modules/host_header.py
@@ -62,7 +62,7 @@ async def interactsh_callback(self, r):
 
     async def finish(self):
         if self.scan.config.get("interactsh_disable", False) is False and self.subdomain_tags:
-            await self.helpers.sleep(5)
+            await self.interactsh_instance.settle()
             try:
                 for r in await self.interactsh_instance.poll():
                     await self.interactsh_callback(r)
diff --git a/bbot/modules/lightfuzz/lightfuzz.py b/bbot/modules/lightfuzz/lightfuzz.py
index 767a3c8690..b1aac883dc 100644
--- a/bbot/modules/lightfuzz/lightfuzz.py
+++ b/bbot/modules/lightfuzz/lightfuzz.py
@@ -352,8 +352,8 @@ async def cleanup(self):
 
     async def finish(self):
         if self.interactsh_instance and self.interactsh_subdomain_tags:
-            self.debug("finish(): sleeping 5s before final interactsh poll")
-            await self.helpers.sleep(5)
+            self.debug("finish(): settling before final interactsh poll")
+            await self.interactsh_instance.settle()
             try:
                 results = await self.interactsh_instance.poll()
                 self.debug(f"finish(): interactsh poll returned {len(results)} interaction(s)")
diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index a122c410c5..714221e69d 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -293,6 +293,10 @@ async def register(self, callback=None):
             self.poll_task = asyncio.create_task(self.poll_loop(callback))
         return "fakedomain.fakeinteractsh.com"
 
+    async def settle(self):
+        # interactions are already queued in-process, so there is no propagation delay to wait out
+        return
+
     async def deregister(self, callback=None):
         await asyncio.sleep(1)
         self.stop = True

From eea93988c4d93acbc6704be6e6f545806a4eef50 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Wed, 2 Sep 2026 08:48:25 -0700
Subject: [PATCH 76/78] perf(test): wait for the interactsh conditions instead
 of a fixed 10s sleep

test_web_interactsh slept a hardcoded 10 seconds between firing the two
out-of-band requests and asserting that both callbacks ran with the right
URL. The sleep was sized for the worst case, so the test always paid the
full budget even though the interactions land much earlier.

Measured against the real interactsh servers, 5 consecutive runs: the last
of the four asserted conditions was satisfied at 4.25s, 1.32s, 3.72s, 3.60s
and 3.56s. The remaining 6 to 9 seconds were pure dead wall time.

Now the test polls the same conditions it later asserts on, at 0.1s, and
still stops at the same 10s deadline. The four asserts are untouched, so a
genuine failure fails exactly as before: verified that with the conditions
never satisfied the loop still runs the full 10.04s and the assertion still
raises, rather than exiting early and masking it. The waiting is bounded by
the identical budget, so this cannot turn a slow interaction into a flake.

Product code is untouched; this is test-side dead time only.

test_web_interactsh 16.4s -> 6.2s across 3 runs. Whole file 21 passed in
14.85s. ruff check and ruff format --check clean.
---
 bbot/test/test_step_1/test_web.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/bbot/test/test_step_1/test_web.py b/bbot/test/test_step_1/test_web.py
index b12d55c5a6..a7143e119c 100644
--- a/bbot/test/test_step_1/test_web.py
+++ b/bbot/test/test_step_1/test_web.py
@@ -345,7 +345,12 @@ def sync_callback(data):
     assert response2.status_code == 200
     assert any(interactsh_domain2.endswith(f"{s}") for s in server_list)
 
-    await asyncio.sleep(10)
+    # poll until every asserted condition holds, capped at the old fixed budget
+    deadline = time.time() + 10
+    while time.time() < deadline:
+        if sync_correct_url and async_correct_url:
+            break
+        await asyncio.sleep(0.1)
 
     data_list = await interactsh_client.poll()
     data_list2 = await interactsh_client2.poll()

From e6bc5ea5c60bf83c57430df1beab93d700467da6 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Wed, 2 Sep 2026 10:11:32 -0700
Subject: [PATCH 77/78] perf(test): serve a static 404 from the test httpserver
 miss path

pytest_httpserver's respond_nohandler builds the miss body from
repr(request) plus a rendering of every registered matcher, so the body is
unique per URL and grows with the handler count. On the webbrute_shortnames
test that is 4318 responses averaging 1136 bytes, 4234 of them distinct.

Brute-force modules establish a baseline from a miss and then diff every
response against it. A miss body that embeds the requested URL never matches
the baseline, so HttpCompare cannot short-circuit and DeepDiffs the full body
every time: 2.02 ms per response, 8.72s summed, 19.8% of execute_fuzz time.
The dump was measurement error, not fidelity. Real servers return a stable
404, which is what the module's filtering logic is written against.

The accumulated assertion goes with it. All three httpserver fixtures call
clear() before check_assertions(), and clear() calls clear_assertions(), so
no-handler assertions were discarded unread and could never fail a test.
Verified directly. Handler errors travel a separate channel
(check_handler_errors) and still surface.

Verified negatively: miss still returns no_handler_status_code (404 and the
403 that bypass403 configures), miss bodies are now stable across URLs,
matched handlers are untouched, and a raising handler still records and
re-raises through check_handler_errors.

webbrute_shortnames 25.7s -> 18.5s over three runs each. Across webbrute,
webbrute_shortnames, iis_shortnames, bypass403 and excavate: 105.7s -> 96.2s
with an identical 71 passed. lightfuzz + generic_ssrf 116 passed, wayback +
reflected_parameters + virtualhost + host_header + dotnetnuke 43 passed,
test_web + test_scan 34 passed.

test_module_http::TestHTTP_URLBlacklist fails locally on the clean tree too,
same 10 passed / 1 failed either way. Not a regression from this change.
---
 bbot/test/conftest.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index 714221e69d..7ebacd6ca1 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -9,6 +9,7 @@
 from pathlib import Path
 from contextlib import suppress
 from pytest_httpserver import HTTPServer
+from werkzeug.wrappers import Response
 from bbot.test.worker import (
     BBOT_TEST_DIR,
     BBOT_TEST_SHARED_DIR,
@@ -32,6 +33,22 @@ class FastShutdownHTTPServer(HTTPServer):
     def thread_target(self) -> None:
         self.server.serve_forever(poll_interval=self.SHUTDOWN_POLL_INTERVAL)
 
+    def respond_nohandler(self, request, extra_message=""):
+        """Serve a static 404 body instead of the upstream diagnostic dump.
+
+        Upstream builds the body from repr(request) plus a rendering of every
+        registered matcher, so the miss body is unique per URL and grows with the
+        handler count. Brute-force modules diff each miss against a baseline miss,
+        and a body that never repeats defeats that comparison: every response looks
+        like a difference and gets DeepDiffed in full. Real servers return a stable
+        404, so the dump was measurement error, not fidelity.
+
+        The accumulated assertion is dropped with it. bbot's httpserver fixtures
+        call clear() before check_assertions(), so no-handler assertions were
+        discarded unread and never failed a test.
+        """
+        return Response("Not Found", self.no_handler_status_code)
+
 
 # silence stdout + trace
 root_logger = logging.getLogger()

From 241397d5de4c33323070e7653a255aa98591ce38 Mon Sep 17 00:00:00 2001
From: Shane Engelman <shane.engelman@blacklanternsecurity.com>
Date: Wed, 2 Sep 2026 10:58:55 -0700
Subject: [PATCH 78/78] fix(test): keep the httpserver miss body empty, not a
 constant string

e6bc5ea5c replaced pytest_httpserver's per-request diagnostic dump with a
static "Not Found" body. That fixed the DeepDiff cost but broke
TestWebParameters_include_count on every python version: expected "3\ttest",
got "2\ttest".

Root cause is content dedup, not the count assertion. base.py:1158 dedups
HTTP_RESPONSEs on (host, port, body_sha256) and explicitly exempts the
empty-body hash e3b0c442..., which is sha256(b""). The upstream dump embedded
the request URL, so every miss body was unique and no miss ever collided. A
constant non-empty body makes all misses hash alike, so the second distinct
miss URL is dropped as duplicate content before excavate sees it. In this test
/validPath and /search are both misses: /validPath was dropped, taking its
HEADER WEB_PARAMETER with it and dropping the test count from 3 to 2.

Serving an empty body keeps the miss body constant across URLs while landing
on the one hash the dedup path already exempts, so no miss is ever dropped.

The perf win is unaffected; the cost was body size and uniqueness, not
content. webbrute_shortnames over three runs each:
  upstream dump  27.70 / 27.22 / 27.93s
  "Not Found"    19.33 / 20.41 / 20.58s
  empty          20.01 / 20.53 / 19.54s

Verified sha256(b"") equals the constant base.py exempts. Negative controls:
miss still returns no_handler_status_code (500 default, 403 when configured),
miss bodies stable across URLs, matched handlers untouched, and a raising
handler still re-raises through check_handler_errors.

web_parameters + excavate + webbrute + webbrute_shortnames + iis_shortnames +
bypass403: 74 passed. lightfuzz + generic_ssrf + virtualhost + host_header +
dotnetnuke + wayback + reflected_parameters: 159 passed. test_web + test_scan:
44 passed.

test_module_http::TestHTTP_URLBlacklist fails locally on the clean tree too,
same assert 4 == 5 either way. Not a regression from this change.
---
 bbot/test/conftest.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/bbot/test/conftest.py b/bbot/test/conftest.py
index 7ebacd6ca1..c1d3819636 100644
--- a/bbot/test/conftest.py
+++ b/bbot/test/conftest.py
@@ -34,20 +34,25 @@ def thread_target(self) -> None:
         self.server.serve_forever(poll_interval=self.SHUTDOWN_POLL_INTERVAL)
 
     def respond_nohandler(self, request, extra_message=""):
-        """Serve a static 404 body instead of the upstream diagnostic dump.
+        """Serve an empty miss body instead of the upstream diagnostic dump.
 
         Upstream builds the body from repr(request) plus a rendering of every
         registered matcher, so the miss body is unique per URL and grows with the
         handler count. Brute-force modules diff each miss against a baseline miss,
         and a body that never repeats defeats that comparison: every response looks
-        like a difference and gets DeepDiffed in full. Real servers return a stable
-        404, so the dump was measurement error, not fidelity.
+        like a difference and gets DeepDiffed in full. The dump was measurement
+        error, not fidelity.
+
+        The body stays empty rather than carrying placeholder text. Modules dedup
+        HTTP_RESPONSEs on (host, port, body_sha256) and base.py exempts the
+        empty-body hash from that check, so an empty miss is the only constant that
+        does not make every miss URL collide as duplicate content.
 
         The accumulated assertion is dropped with it. bbot's httpserver fixtures
         call clear() before check_assertions(), so no-handler assertions were
         discarded unread and never failed a test.
         """
-        return Response("Not Found", self.no_handler_status_code)
+        return Response("", self.no_handler_status_code)
 
 
 # silence stdout + trace