perf(ci): cut test suite wall time via teardown, poll, and dep-install fixes

.github/workflows/distro_tests.yml

@@ -51,7 +51,10 @@ jobs:
           uv python install 3.12
           uv python pin 3.12
           uv sync --group dev
-          uv run pytest --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO .
+          uv run pytest -n $(uv run python bbot/test/worker_count.py) --dist loadgroup --reruns 2 --exitfirst -o timeout_func_only=true --timeout 1200 --disable-warnings --log-cli-level=INFO \
+            bbot/test/test_step_1/test_e2e.py \
+            bbot/test/test_step_1/test_depsinstaller.py \
+            bbot/test/test_step_1/test_command.py
       - name: Upload Debug Logs
         if: always()
         uses: actions/upload-artifact@v7

bbot/core/config/models.py

@@ -294,6 +294,7 @@ class WebConfig(BaseModel):
     max_sleep_interval_429: Optional[int] = Field(default=None, alias="429_max_sleep_interval")
     debug: Optional[bool] = None
     http_max_redirects: Optional[int] = None
+    http_compare_max_differing_lines: Optional[int] = None
     ssl_verify_target: Optional[bool] = None
     ssl_verify_infrastructure: Optional[bool] = None
 

bbot/core/core.py

@@ -64,21 +64,35 @@ def _prep_multiprocessing(self):
     def home(self):
         return Path(self.config["home"]).expanduser().resolve()
 
+    @property
+    def deps_home(self):
+        """
+        Install state, downloaded tools and libs are keyed on content, not on the scan,
+        so concurrent scans can share them. BBOT_SHARED_DEPS_DIR points them at one
+        location while everything scan-specific stays under home.
+        """
+        shared = os.environ.get("BBOT_SHARED_DEPS_DIR", "").strip()
+        if shared:
+            return Path(shared).expanduser().resolve()
+        return self.home
+
     @property
     def cache_dir(self):
-        return self.home / "cache"
+        return self.deps_home / "cache"
 
     @property
     def tools_dir(self):
-        return self.home / "tools"
+        return self.deps_home / "tools"
 
     @property
     def temp_dir(self):
-        return self.home / "temp"
+        # BBOT_TEMP is the scratch dir for dependency builds and is baked into the
+        # module hash, so it belongs with the other install-once dirs.
+        return self.deps_home / "temp"
 
     @property
     def lib_dir(self):
-        return self.home / "lib"
+        return self.deps_home / "lib"
 
     @property
     def scans_dir(self):