From 8e6e963784dffd021d6a13a89368346f1b2946af Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 11 Aug 2026 14:35:46 +0900 Subject: [PATCH 1/4] fix: retry Strix provider tool protocol failures --- scripts/ci/strix_quick_gate.sh | 19 +++++++++++++++++++ scripts/ci/test_strix_quick_gate.sh | 27 +++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 0f37f3460..7486ab22f 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2939,6 +2939,15 @@ is_llm_token_limit_error() { return 1 } +# Strix's agent SDK can reject a provider response before it produces scan +# evidence when the model emits a tool call that the selected agent does not +# expose. Treat this exact SDK/provider failure as retryable so a configured +# fallback model can complete the security scan; it is not evidence from the +# target repository and must never be treated as a vulnerability. +is_model_tool_protocol_error() { + grep -Eiq 'agents\.exceptions\.ModelBehaviorError:[[:space:]]*Tool execute not found in agent strix' "$STRIX_LOG" +} + # Detect whether the strix log contains evidence of infrastructure-level # errors (timeout, rate-limit, transport failures) that indicate the scan # was interrupted or incomplete. Used as a guard to prevent the @@ -2960,6 +2969,10 @@ has_detected_infrastructure_error() { return 0 fi + if is_model_tool_protocol_error; then + return 0 + fi + if is_midstream_fallback_error; then return 0 fi @@ -3838,6 +3851,12 @@ is_model_retryable_error() { return 0 fi + if is_model_tool_protocol_error; then + # A provider/model tool-contract failure is recoverable with a distinct + # configured model, but it is not a clean scan result. + return 0 + fi + if is_timeout_error; then # Process and provider timeouts are not clean evidence, but they are # recoverable across distinct fallback models. Strict provider-signal diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 7343c06ac..45a51bbb2 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -727,6 +727,8 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/scripts/ci/run_opencode_review_model_pool.sh" "skipping remaining attempts for this model" "opencode review skips same-model retries after context-window overflow" assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "exceeded your current quota" "strix wrapper neutralizes quota-only provider failures without vulnerability reports" assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure" + assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "is_model_tool_protocol_error" "strix quick gate retries agent tool-protocol provider failures" + assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "ModelBehaviorError:[[:space:]]*Tool execute not found in agent strix" "strix quick gate identifies the exact unsupported tool response" assert_file_contains "$workflow_file" 'timeout-minutes: 325' "opencode review target contains evidence, the bounded long-review pool, publication, Noema handoff, and cleanup overhead" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue" assert_file_contains "$workflow_file" 'timeout-minutes: 205' "opencode model pool preserves full-hour candidates within a bounded provider-pool window" @@ -3564,6 +3566,22 @@ REPORT ;; esac ;; + tool-protocol-fallback-success) + case "${STRIX_LLM:-}" in + vertex_ai/tool-protocol-primary) + echo "agents.exceptions.ModelBehaviorError: Tool execute not found in agent strix" + exit 1 + ;; + vertex_ai/fallback-one) + echo "scan ok after tool-protocol fallback" + exit 0 + ;; + *) + echo "Error: tool-protocol fallback path unexpected (${STRIX_LLM:-})" >&2 + exit 26 + ;; + esac + ;; openai-primary-quota-fallback-success) case "${STRIX_LLM:-}" in openai/quota-primary) @@ -9329,6 +9347,15 @@ run_gate_case_allow_provider_signal "vertex-primary-midstream-fallback-success" "vertex_ai/midstream-primary|vertex_ai/fallback-one" \ "|" +run_gate_case "tool-protocol-fallback-success" \ + "vertex_ai/tool-protocol-primary" \ + "vertex_ai/fallback-one" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ + "2" \ + "vertex_ai/tool-protocol-primary|vertex_ai/fallback-one" \ + "|" + run_gate_case_allow_provider_signal "vertex-primary-midstream-retry-same-model-success" \ "vertex_ai/retry-midstream-primary" \ "vertex_ai/fallback-one vertex_ai/fallback-two" \ From 6273bdbecfef2c8fca4fc5115e001e7ff82ed506 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 11 Aug 2026 14:42:27 +0900 Subject: [PATCH 2/4] test: expose Strix tool fallback regression filter --- scripts/ci/test_strix_quick_gate.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 45a51bbb2..53344256c 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -5843,6 +5843,16 @@ run_filtered_gate_case_if_requested() { input-file-root-override-precedence) run_input_file_root_override_takes_precedence_over_runner_temp_case ;; + tool-protocol-fallback-success) + run_gate_case "tool-protocol-fallback-success" \ + "vertex_ai/tool-protocol-primary" \ + "vertex_ai/fallback-one" \ + "0" \ + "REGEX:Strix quick scan succeeded with fallback model 'vertex_ai/fallback-one' in [0-9]+s\\." \ + "2" \ + "vertex_ai/tool-protocol-primary|vertex_ai/fallback-one" \ + "|" + ;; vertex-without-llm-api-key) run_vertex_without_llm_api_key_case ;; From b4e60dcec7abea2466b5f1d35b7f42e23313ebc8 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Tue, 11 Aug 2026 16:18:07 +0900 Subject: [PATCH 3/4] fix: classify all Strix tool protocol failures --- scripts/ci/strix_quick_gate.sh | 2 +- scripts/ci/test_strix_quick_gate.sh | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/scripts/ci/strix_quick_gate.sh b/scripts/ci/strix_quick_gate.sh index 7486ab22f..e82eabe8d 100755 --- a/scripts/ci/strix_quick_gate.sh +++ b/scripts/ci/strix_quick_gate.sh @@ -2945,7 +2945,7 @@ is_llm_token_limit_error() { # fallback model can complete the security scan; it is not evidence from the # target repository and must never be treated as a vulnerability. is_model_tool_protocol_error() { - grep -Eiq 'agents\.exceptions\.ModelBehaviorError:[[:space:]]*Tool execute not found in agent strix' "$STRIX_LOG" + grep -Eiq 'agents\.exceptions\.ModelBehaviorError:[[:space:]]*Tool [[:alnum:]_]+ not found in agent strix' "$STRIX_LOG" } # Detect whether the strix log contains evidence of infrastructure-level diff --git a/scripts/ci/test_strix_quick_gate.sh b/scripts/ci/test_strix_quick_gate.sh index 53344256c..29a81613f 100755 --- a/scripts/ci/test_strix_quick_gate.sh +++ b/scripts/ci/test_strix_quick_gate.sh @@ -728,7 +728,7 @@ assert_opencode_review_uses_codegraph_and_gpt5_fallback() { assert_file_contains "$REPO_ROOT/.github/workflows/strix.yml" "exceeded your current quota" "strix wrapper neutralizes quota-only provider failures without vulnerability reports" assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "billing details" "strix quick gate classifies provider quota starvation as infrastructure" assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "is_model_tool_protocol_error" "strix quick gate retries agent tool-protocol provider failures" - assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "ModelBehaviorError:[[:space:]]*Tool execute not found in agent strix" "strix quick gate identifies the exact unsupported tool response" + assert_file_contains "$REPO_ROOT/scripts/ci/strix_quick_gate.sh" "ModelBehaviorError:[[:space:]]*Tool [[:alnum:]_]+ not found in agent strix" "strix quick gate identifies unsupported agent tool responses" assert_file_contains "$workflow_file" 'timeout-minutes: 325' "opencode review target contains evidence, the bounded long-review pool, publication, Noema handoff, and cleanup overhead" assert_file_contains "$workflow_file" 'timeout-minutes: 12' "opencode evidence preparation fails closed before it ties up the review queue" assert_file_contains "$workflow_file" 'timeout-minutes: 205' "opencode model pool preserves full-hour candidates within a bounded provider-pool window" @@ -3570,6 +3570,7 @@ REPORT case "${STRIX_LLM:-}" in vertex_ai/tool-protocol-primary) echo "agents.exceptions.ModelBehaviorError: Tool execute not found in agent strix" + echo "agents.exceptions.ModelBehaviorError: Tool agent_finish not found in agent strix" exit 1 ;; vertex_ai/fallback-one) From 87d2519b7ffbd6270a95400cde62a296ffea0ab3 Mon Sep 17 00:00:00 2001 From: Seongho Bae Date: Thu, 13 Aug 2026 12:38:27 +0900 Subject: [PATCH 4/4] docs(strix): record exact tool-protocol fallback and ADRs Cite NIST SP 800-53 and the Agents SDK for retrying only ModelBehaviorError missing-tool signals. Add ARCHITECTURE.md diagram. Isolate Darwin installer tests on the linux x86_64 path. --- AGENTS.md | 2 +- ARCHITECTURE.md | 78 +++++++++++++++++++ CHANGELOG.md | 1 + CLAUDE.md | 4 + .../doctoring/strix-tool-protocol-fallback.md | 27 +++++++ ...st_materialize_base_python_requirements.py | 10 +++ 6 files changed, 121 insertions(+), 1 deletion(-) create mode 100644 ARCHITECTURE.md create mode 100644 docs/doctoring/strix-tool-protocol-fallback.md diff --git a/AGENTS.md b/AGENTS.md index 688b33035..b65862b78 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -1,4 +1,4 @@ # AGENTS.md — ContextualWisdomLab .github -> **Agents: read the master context FIRST.** Before any work, read [`docs/CWL-MASTER-CONTEXT.md`](docs/CWL-MASTER-CONTEXT.md) (mission · naruon-as-platform + inter-component UML · cross-cutting disciplines · conventions · roadmap · current state), the live **GitHub Project #1** (work/roadmap source of truth), the full spec **ContextualWisdomLab/naruon#974**, and operate the Project per [`docs/agent-github-project-protocol.md`](docs/agent-github-project-protocol.md). The repo/Project — not any private agent memory — is the source of truth. +> **Agents: read the master context FIRST.** Before any work, read [`docs/CWL-MASTER-CONTEXT.md`](docs/CWL-MASTER-CONTEXT.md) (mission · naruon-as-platform + inter-component UML · cross-cutting disciplines · conventions · roadmap · current state), [`ARCHITECTURE.md`](ARCHITECTURE.md) (control-plane context and Strix tool-protocol fallback), the live **GitHub Project #1** (work/roadmap source of truth), the full spec **ContextualWisdomLab/naruon#974**, and operate the Project per [`docs/agent-github-project-protocol.md`](docs/agent-github-project-protocol.md). The repo/Project — not any private agent memory — is the source of truth. Strix retries only the exact OpenAI Agents SDK missing-tool signal; it never treats that provider failure as a clean scan. diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md new file mode 100644 index 000000000..60b21015d --- /dev/null +++ b/ARCHITECTURE.md @@ -0,0 +1,78 @@ +# Architecture — ContextualWisdomLab `.github` + +This repository is the organization control plane. It is not naruon and it +does not own product data. Sibling products remain standalone modules; this +repo publishes org profile assets, reusable required workflows, and the +review/merge schedulers those products consume. + +## System context + +```mermaid +flowchart LR + Buyer["Commercial buyer / reviewer"] + Agents["Agents on AGENTS.md"] + Project["GitHub Project #1"] + Hub["This repo: org .github"] + Products["Owned products
naruon · orchestrator · engines"] + Runner["Required workflows in each repo context"] + + Buyer --> Hub + Agents --> Project + Agents --> Hub + Project --> Hub + Hub --> Runner + Runner --> Products + Products -->|"standalone or as module"| Buyer +``` + +## Strix tool-protocol fallback + +```mermaid +sequenceDiagram + participant Gate as strix_quick_gate + participant Model as Current model + participant Next as Distinct configured model + participant Art as Vulnerability artifacts + + Gate->>Model: scan exact head + alt ModelBehaviorError Tool NAME not found in agent strix + Model-->>Gate: retryable provider protocol failure + Gate->>Next: distinct fallback model + Next-->>Gate: scan evidence + else threshold artifacts already present + Art-->>Gate: fail closed + else fallback exhausted + Gate-->>Gate: fail closed + end +``` + +## Trust boundaries + +- Required review workflows execute **base-branch** scripts. A PR that edits + those workflows cannot widen its own `pull_request_target` token. +- Reviewer agents stay `edit: deny`. They judge; they do not implement. +- Strix retries only the exact OpenAI Agents SDK missing-tool signal for + agent `strix`. The signal is not a clean scan. Severity thresholds, + credentials, and publication stay unchanged. +- Logs and review receipts redact credentials. They do not mask + operational PII that the control plane must process. +- LLM and scheduled agents bind `NVIDIA_NIM_API_KEY` (env may be + `NVIDIA_API_KEY`). They never use `COPILOT_GITHUB_TOKEN`. Existing + review-agent key schemes stay unchanged. + +## Quality gates + +`scripts/ci/` ships with 100% statement/branch coverage and 100% docstrings. +CI installs Python tools only with `pip install --require-hashes`. Contract +tests pin workflow structure and governance prose so drift fails closed. + +## Related durable documents + +- [`docs/CWL-MASTER-CONTEXT.md`](docs/CWL-MASTER-CONTEXT.md) — mission and + ecosystem. +- [`docs/agent-github-project-protocol.md`](docs/agent-github-project-protocol.md) + — Project #1 operation. +- [`PR_GOVERNANCE_AUDIT.md`](PR_GOVERNANCE_AUDIT.md) — live review/merge + contract. +- [`docs/doctoring/strix-tool-protocol-fallback.md`](docs/doctoring/strix-tool-protocol-fallback.md) + — tool-protocol fallback decision and APA 7th citations. diff --git a/CHANGELOG.md b/CHANGELOG.md index bf30091dd..c4739c821 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ Semantic Versioning where the repository publishes a release. ### Fixed +- Retried only the exact OpenAI Agents SDK `ModelBehaviorError: Tool not found in agent strix` signal onto a distinct configured model, without treating that provider failure as a clean scan. - Bounded the Strix quality self-test's deterministic timeout fixtures to 3-second process and 5-second fake-sleep budgets so exact-head policy evidence completes inside the existing job limit without changing production Strix scanner timeouts, providers, credentials, or review semantics. - Allowed commas and ASCII parentheses in the bounded Strix changed-file path policy so legal tracked Packrat fixtures can receive exact-head security analysis, while rejecting raw `..` components before normalization and keeping controls, backslashes, whitespace ambiguity, and shell punctuation fail-closed. - Bound each review-agent invocation key to the wrapper's complete canonical payload, including the base branch and requesting actor; altered fields with a valid-format key now fail before durable-leader election or forwarding, and wrapper write permission is job-scoped. diff --git a/CLAUDE.md b/CLAUDE.md index 1c7bdb2f6..acb151185 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -126,3 +126,7 @@ repeatable compile command. cross-repo references as `owner/repo#num` or full URLs; durable knowledge in the repo/Project, not private memory; one roadmap phase at a time) are defined in `docs/CWL-MASTER-CONTEXT.md` §7 and apply here. +- **Strix tool-protocol fallback** retries only + `agents.exceptions.ModelBehaviorError: Tool not found in agent strix`. + That signal is infrastructure, not a clean scan. See `ARCHITECTURE.md` and + `docs/doctoring/strix-tool-protocol-fallback.md`. diff --git a/docs/doctoring/strix-tool-protocol-fallback.md b/docs/doctoring/strix-tool-protocol-fallback.md new file mode 100644 index 000000000..594be186a --- /dev/null +++ b/docs/doctoring/strix-tool-protocol-fallback.md @@ -0,0 +1,27 @@ +# Strix tool-protocol fallback + +## Incident and buyer impact + +Strix can exit before producing security evidence when the selected model +emits a tool call the `strix` agent does not expose: + +`agents.exceptions.ModelBehaviorError: Tool execute not found in agent strix` + +The required check then failed as if the target repository were insecure, +and configured fallback models never ran. + +## Decision + +Classify only that exact exception class, missing-tool phrase, and agent +name `strix` as retryable infrastructure. Fallback to a distinct configured +model. Vulnerability artifacts and fallback exhaustion remain fail-closed. +Generic application errors are not matched. + +## References + +National Institute of Standards and Technology. (2020). *Security and +privacy controls for information systems and organizations* (NIST Special +Publication 800-53 Rev. 5). https://doi.org/10.6028/NIST.SP.800-53r5 + +OpenAI. (2025). *OpenAI Agents SDK*. +https://openai.github.io/openai-agents-python/ diff --git a/tests/test_materialize_base_python_requirements.py b/tests/test_materialize_base_python_requirements.py index 8a383f0c2..10f682b3e 100644 --- a/tests/test_materialize_base_python_requirements.py +++ b/tests/test_materialize_base_python_requirements.py @@ -30,6 +30,13 @@ def _created_tool_directory(path: Path) -> str: return str(path) +def _force_linux_x86_64_installer(monkeypatch: pytest.MonkeyPatch) -> None: + """Exercise the installer path that GitHub-hosted linux x86_64 runners use.""" + monkeypatch.setattr(materializer.sys, "platform", "linux") + monkeypatch.setattr(materializer.platform, "machine", lambda: "x86_64") + materializer._install_trusted_uv.cache_clear() + + def test_materializes_only_regular_hash_locks_from_exact_base(tmp_path: Path) -> None: """A PR-modified lock cannot enter the networked coverage image build context.""" repo = tmp_path / "repo" @@ -644,6 +651,7 @@ def test_install_trusted_uv_verifies_version_and_caches_path( tmp_path: Path, monkeypatch: pytest.MonkeyPatch ) -> None: """The installer writes one executable, verifies its version, and caches it.""" + _force_linux_x86_64_installer(monkeypatch) tool_dir = tmp_path / "uv" monkeypatch.setattr( materializer.tempfile, @@ -690,6 +698,7 @@ def test_install_trusted_uv_rejects_version_process_failures( failure: OSError | subprocess.TimeoutExpired, ) -> None: """A missing or hung downloaded executable is removed and rejected.""" + _force_linux_x86_64_installer(monkeypatch) tool_dir = tmp_path / "uv" monkeypatch.setattr( materializer.tempfile, @@ -721,6 +730,7 @@ def test_install_trusted_uv_rejects_wrong_version_or_exit_status( completed: subprocess.CompletedProcess[bytes], ) -> None: """Unexpected version output or a nonzero status cannot satisfy the pin.""" + _force_linux_x86_64_installer(monkeypatch) tool_dir = tmp_path / f"uv-{completed.returncode}-{len(completed.stdout)}" monkeypatch.setattr( materializer.tempfile,