Fix Jev fast path after automatic screenshots

docs/architecture.md

@@ -1042,7 +1042,8 @@ The existing AX walk supplies at most 24 structured controls with its own refs;
 page-authored ref strings are never parsed. Internal snapshots are stripped from
 public tool results and diagnostics. Identity, document, form structure, options,
 value and occlusion are checked again before dispatch. Frames, shadow roots,
-credential/file fields and unsupported actions fall back. Field values come from
+unsupported actions and pages containing credential, payment, OTP or file controls
+fall back as a whole. Field values come from
 the active provider only after a confident fill action/target is selected. The
 first uncached fill uses a second Jev request to map those prepared values;
 clicks, completion candidates and fallbacks do not prepare text. Cached values
@@ -1064,3 +1065,10 @@ run cancellation and model cost limits apply. `done` remains an active-model
 operation using existing evidence checks. RAG, skill routing and direct watch
 poll optimization are not part of this integration. See `test/jev/README.md` for
 benchmark protocol and its unverified live-performance status.
+
+Initial-page and automatic browser screenshots do not make the AX-only path
+ineligible, and their pixels are never included in a Jev request. A current user
+attachment, explicit screenshot-tool result or unknown non-text input routes that
+decision to the active provider. The fast path may resume after that provider has
+consumed the input. Trace exports render Jev routing, fallback and usage metadata,
+including skip reasons, without exporting evidence.

docs/privacy-and-data-flow.md

@@ -668,8 +668,13 @@ or scheduler settings do not enable them. Fast classification sends bounded
 request context; fast browser decisions send the task, up to 24 structured AX
 controls, observed options and bounded prepared field values. These may include
 ordinary personal text explicitly supplied for a form. Credential-related tasks
-and credential/file controls are excluded from the fast path. Redaction remains
-best effort. The active chat provider prepares free text and gives the final
-answer; Jev receives neither screenshots nor full conversation history for
-browser decisions. Separate requests use the same pinned model, cost accounting
-and untrusted-data boundaries, with a one-second deadline and zero retries.
+and pages containing credential, payment, OTP or file controls are excluded from
+the fast path as a whole. Redaction remains best effort. Initial and automatic
+browser screenshots do not disable the AX-only path, but their pixels are never
+sent to Jev. A current user attachment, explicit screenshot-tool result or unknown
+non-text input keeps that decision on the active chat provider. That provider also
+prepares free text and gives the final answer; Jev receives neither screenshots nor
+full conversation history for browser decisions. Separate requests use the same
+pinned model, cost accounting and untrusted-data boundaries, with a one-second
+deadline and zero retries. Exported traces show Jev decisions and skip reasons but
+never the bounded request evidence.

src/chrome/src/agent/agent.js

@@ -1,4 +1,4 @@
-import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, JevFastSession } from './systemone-fast.js';
+import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';
 import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js';
 import { createSystemOneJudge, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';
 import { SOCIAL_PLATFORMS, socialPublicationApiPlatform, normalizePublicationContract, publicationProgress, exactPublicationText, publicationMediaMatches, publicationContractMessages, publicationAuditMessages, publicationAuditAccepted } from './social-publish-contract.js';
@@ -6870,13 +6870,23 @@ export class Agent extends LoopDetector {
 
   async _maybeJevFastTurn(tabId, task, messages, mode, allowed, provider, costState, runOptions = {}, recovery = null) {
     const context = this.systemOneContext(tabId);
-    if (!['act', 'dev'].includes(mode) || recovery || runOptions.cloudRun || this.selectionGroundingScopes.has(tabId) || this._isStandaloneChatRun(runOptions)
-      || messages.some(message => Array.isArray(message?.content) && message.content.some(block => block?.type !== 'text'))
-      || this._checkAbort(tabId) || /log.?in|sign.?in|password|parola|giriş|oturum|credential|api.?key|secret/i.test(task)) return null;
+    if (!['act', 'dev'].includes(mode) || this._checkAbort(tabId)) return null;
     let session;
     try {
       const settings = await this._jevSettings();
-      if (!settings?.systemOneFastBrowser || !allowed.has('get_accessibility_tree') || !context.isCurrent()) return null;
+      if (!settings?.systemOneFastBrowser || !context.isCurrent()) return null;
+      let skipReason = '';
+      if (recovery) skipReason = 'recovery_turn';
+      else if (runOptions.cloudRun) skipReason = 'cloud_run';
+      else if (this.selectionGroundingScopes.has(tabId)) skipReason = 'selection_grounded';
+      else if (this._isStandaloneChatRun(runOptions)) skipReason = 'standalone_chat';
+      else if (jevVisualInputRequiresMainModel(messages)) skipReason = 'current_visual_input';
+      else if (/log.?in|sign.?in|password|passwd|passcode|\b(?:otp|token|secret)\b|one.?time.?code|parola|giriş|oturum|credential|api.?key/i.test(task)) skipReason = 'sensitive_task';
+      else if (!allowed.has('get_accessibility_tree')) skipReason = 'tool_policy';
+      if (skipReason) {
+        this.recordSystemOneVerdict(tabId, { decision: 'skip', reason: skipReason }, context);
+        return null;
+      }
       this._jevSessions ??= new Map();
       session = this._jevSessions.get(tabId);
       if (!session) { session = new JevFastSession(); this._jevSessions.set(tabId, session); }
@@ -6891,6 +6901,7 @@ export class Agent extends LoopDetector {
         this.recordSystemOneVerdict(tabId, { decision: 'fallback', reason }, context);
         return null;
       };
+      if (session.snapshot.hasSensitiveControls === true) return fallback('sensitive_controls');
       const taskText = String(task).slice(0, 4000);
       const cached = session.valueContext === this._jevValueContext(taskText, session.snapshot) ? session.values || [] : [];
       let request = buildJevBrowserRequest(taskText, session.snapshot, cached);

src/chrome/src/agent/systemone-fast.js

@@ -2,6 +2,49 @@ import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js'
 export const JEV_CLASSIFIER_THRESHOLD = .85;
 export const JEV_BROWSER_THRESHOLD = .90;
 export const JEV_FAST_KEYS = ['systemOneEnabled', 'typesafeApiKey', 'systemOneFastClassifications', 'systemOneFastBrowser'];
+
+function messageText(message) {
+  if (typeof message?.content === 'string') return message.content;
+  if (!Array.isArray(message?.content)) return '';
+  return message.content
+    .filter(block => block?.type === 'text' && typeof block.text === 'string')
+    .map(block => block.text)
+    .join('\n');
+}
+
+function hasNonTextBlock(message) {
+  return Array.isArray(message?.content)
+    && message.content.some(block => block?.type !== 'text');
+}
+
+// Only a visual input that the main model has not consumed yet blocks Jev.
+// Initial and automatic browser captures are auxiliary context; Jev makes its
+// decision from a fresh bounded AX snapshot and never receives those pixels.
+// User attachments and explicit screenshot-tool results still require the main
+// model for visual reasoning. Looking only after the latest assistant message
+// lets Jev resume after that model has consumed an explicit visual observation.
+export function jevVisualInputRequiresMainModel(messages) {
+  if (!Array.isArray(messages)) return false;
+  let start = 0;
+  for (let index = messages.length - 1; index >= 0; index -= 1) {
+    if (messages[index]?.role === 'assistant') {
+      start = index + 1;
+      break;
+    }
+  }
+  for (let index = start; index < messages.length; index += 1) {
+    const message = messages[index];
+    if (!hasNonTextBlock(message)) continue;
+    const text = messageText(message);
+    if (text.includes('[UNTRUSTED USER ATTACHMENTS')) return true;
+    if (/\[UNTRUSTED SCREENSHOT[^\]]*Screenshot from your [^\]]+ call\./i.test(text)) return true;
+    if (/\[UNTRUSTED SCREENSHOT[^\]]*Capture ID:/i.test(text)) continue;
+    if (/\[UNTRUSTED CAPTURE[^\]]*Capture ID:/i.test(text)) continue;
+    return true;
+  }
+  return false;
+}
+
 export function confidentChoice(answer, threshold) {
   return answer?.type === 'choice' && typeof answer.confidence === 'number' && answer.confidence >= threshold
     && typeof answer.probabilities?.[answer.choice] === 'number' && answer.probabilities[answer.choice] >= threshold
@@ -10,7 +53,7 @@ export function confidentChoice(answer, threshold) {
 const question = (instructions, criteria) => ({ type: 'choice', instructions, criteria });
 const NONE = { none: 'No supported target; use the main model.' };
 export function buildJevBrowserRequest(task, snapshot, values = []) {
-  if (!snapshot || !Array.isArray(snapshot.controls) || !snapshot.documentToken || !snapshot.structure) return null;
+  if (!snapshot || snapshot.hasSensitiveControls === true || !Array.isArray(snapshot.controls) || !snapshot.documentToken || !snapshot.structure) return null;
   const controls = snapshot.controls.slice(0, 24);
   const targets = kind => Object.fromEntries(controls.filter(c => c.kinds.includes(kind)).map(c => [c.ref, `Observed ${kind} target ${c.ref} in state.controls.`]));
   const choices = { click: 'Click a visible control or link, including a requested final submit/save/send.', fill: 'Fill fields required by the user task. Values can be prepared after this action is selected.', select: 'Select an observed native option.', check: 'Set a checkbox state.', scroll_down: 'Scroll down to reveal controls.', scroll_up: 'Scroll up.', wait: 'Wait for the page to settle.', done: 'Candidate completion: ask the main model to verify evidence and respond.', fallback: 'Unsupported, ambiguous, visual, iframe, shadow, upload, keyboard, code or WebMCP work: use the main model.' };
@@ -101,7 +144,7 @@ export class JevFastSession {
   get fallbackBlocked() { return this.fallbackCount >= 2; }
   recordFallback() { this.fallbackCount++; this.queue = []; }
   observe(snapshot) {
-    const context = JSON.stringify([snapshot?.documentToken, snapshot?.pageUrl, snapshot?.structure, snapshot?.progress]);
+    const context = JSON.stringify([snapshot?.documentToken, snapshot?.pageUrl, snapshot?.structure, snapshot?.progress, snapshot?.hasSensitiveControls === true]);
     if (context !== this.fallbackContext) {
       this.fallbackCount = 0;
       this.completionCandidate = false;
@@ -111,7 +154,8 @@ export class JevFastSession {
     else if (this.pending) this.noProgress = 0;
     this.pending = false;
     if (this.noProgress >= 2) this.disabled = true;
-    if (this.snapshot?.structure !== snapshot?.structure || this.snapshot?.documentToken !== snapshot?.documentToken) this.queue = [];
+    if (this.snapshot?.structure !== snapshot?.structure || this.snapshot?.documentToken !== snapshot?.documentToken
+        || (this.snapshot?.hasSensitiveControls === true) !== (snapshot?.hasSensitiveControls === true)) this.queue = [];
     this.snapshot = snapshot;
   }
   dispatched(result) {

src/chrome/src/agent/trace-export.js

@@ -352,6 +352,21 @@ export function tracesToMarkdown(runsWithEvents, {
         const details = [oneLine(d.context), oneLine(d.visionRoute), oneLine(d.model), oneLine(d.captureId)]
           .filter(Boolean).join(' · ');
         md += `- 👁 Vision route${details ? `: ${details}` : ''}${d.fallbackReason ? ` · fallback=${oneLine(d.fallbackReason)}` : ''}\n`;
+      } else if (ev.kind === 'note' && d.note === 'system_one') {
+        const metadata = d.extra || {};
+        const usage = metadata.usage || {};
+        const inputTokens = Number(usage.prompt_tokens ?? usage.input_tokens);
+        const outputTokens = Number(usage.completion_tokens ?? usage.output_tokens);
+        const details = [
+          metadata.reason ? `reason=${oneLine(metadata.reason)}` : '',
+          metadata.model ? `model=${oneLine(metadata.model)}` : '',
+          Number.isFinite(metadata.latencyMs) ? `${metadata.latencyMs} ms` : '',
+          Number.isFinite(metadata.estimatedCostUsd) ? `$${metadata.estimatedCostUsd.toFixed(6)}` : '',
+          Number.isFinite(inputTokens) || Number.isFinite(outputTokens)
+            ? `${Number.isFinite(inputTokens) ? inputTokens : 0} in / ${Number.isFinite(outputTokens) ? outputTokens : 0} out`
+            : '',
+        ].filter(Boolean).join(' · ');
+        md += `- ⚡ Jev: ${oneLine(metadata.decision || 'event')}${details ? ` · ${details}` : ''}\n`;
       } else if (ev.kind === 'note' && d.note === 'vision_status') {
         const status = d.extra || {};
         const progress = Number.isFinite(Number(status.progress)) ? `${Math.round(Number(status.progress))}%` : '';

src/chrome/src/content/accessibility-tree.js

@@ -715,12 +715,18 @@
   // parses page-authored ref strings and cannot target frames or shadow roots.
   let jevCollector = null;
   let lastJevSnapshot = null;
+  let jevSensitiveControlSeen = false;
   function jevControl(el) {
     if (window.top !== window || el.getRootNode() !== document || !isInteractive(el)) return null;
     const tag = el.tagName.toLowerCase();
     const type = String(el.type || '').toLowerCase();
     const name = String(getAccessibleName(el) || '').slice(0, 120);
-    if (['password', 'file', 'hidden'].includes(type) || /password|passwd|api.?key|token|secret|credit.?card|cvv|cvc/i.test([name, el.name, el.id, el.autocomplete].join(' '))) return null;
+    const identity = [name, el.name, el.id, el.autocomplete].join(' ');
+    if (['password', 'file', 'hidden'].includes(type)
+        || /\b(?:password|passwd|passcode|pin|otp|token|secret|cvv|cvc|csc|ssn|iban)\b|\bcc-(?:name|given-name|additional-name|family-name|number|exp|exp-month|exp-year|csc|type)\b|api.?key|one[-_\s]?time(?:[-_\s]?code)?|verification.?code|security.?code|auth(?:entication)?.?code|credit.?card|card.?number|social.?security|(?:routing|account).?number/i.test(identity)) {
+      jevSensitiveControlSeen = true;
+      return null;
+    }
     const role = getRole(el);
     const kinds = [];
     if (tag === 'select') kinds.push('select');
@@ -741,15 +747,18 @@
     return control;
   }
   function collectJevControl(el) {
-    if (!jevCollector || jevCollector.size >= 24) return;
-    try { const control = jevControl(el); if (control) jevCollector.set(control.ref, control); } catch { /* Optional inventory must not break the normal AX reader. */ }
+    if (!jevCollector) return;
+    try {
+      const control = jevControl(el);
+      if (control && jevCollector.size < 24) jevCollector.set(control.ref, control);
+    } catch { /* Optional inventory must not break the normal AX reader. */ }
   }
   function jevSnapshot() { return lastJevSnapshot; }
   function jevValidate(binding) {
     if (!binding || binding.pageUrl !== location.href) return false;
     generateAccessibilityTree('interactive', 10, 3500);
     const snapshot = lastJevSnapshot;
-    if (!snapshot || snapshot.structure !== binding.structure) return false;
+    if (!snapshot || snapshot.hasSensitiveControls === true || snapshot.structure !== binding.structure) return false;
     const target = snapshot.controls.find(c => c.ref === binding.ref);
     const el = lookup(binding.ref);
     if (!target || target.disabled || target.signature !== binding.signature || !el || el.getRootNode() !== document) return false;
@@ -1452,6 +1461,7 @@
   function generateAccessibilityTree(filter, maxDepth, maxChars, refId, page, expectedTreeRevision) {
     jevCollector = filter === 'interactive' && !refId && (!page || page === 1) ? new Map() : null;
     lastJevSnapshot = null;
+    jevSensitiveControlSeen = false;
     try {
       ensureRefScope();
       const effFilter = filter || 'all';
@@ -1723,7 +1733,7 @@
       if (jevCollector) {
         const controls = [...jevCollector.values()];
         const structure = fingerprintTreeContent(JSON.stringify(controls.map(({ value, checked, signature, ...control }) => control)));
-        lastJevSnapshot = { controls, structure, progress: fingerprintTreeContent(JSON.stringify(controls)) + ':' + Math.round(scrollY) };
+        lastJevSnapshot = { controls, structure, progress: fingerprintTreeContent(JSON.stringify(controls)) + ':' + Math.round(scrollY), hasSensitiveControls: jevSensitiveControlSeen };
       }
       jevCollector = null;
     }

src/firefox/src/agent/agent.js

@@ -1,4 +1,4 @@
-import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, JevFastSession } from './systemone-fast.js';
+import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js';
 import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js';
 import { createSystemOneJudge, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js';
 import { firefoxBidi } from '../bidi/client.js';
@@ -6654,13 +6654,23 @@ export class Agent extends LoopDetector {
 
   async _maybeJevFastTurn(tabId, task, messages, mode, allowed, provider, costState, runOptions = {}, recovery = null) {
     const context = this.systemOneContext(tabId);
-    if (!['act', 'dev'].includes(mode) || recovery || runOptions.cloudRun || this.selectionGroundingScopes.has(tabId) || this._isStandaloneChatRun(runOptions)
-      || messages.some(message => Array.isArray(message?.content) && message.content.some(block => block?.type !== 'text'))
-      || this._checkAbort(tabId) || /log.?in|sign.?in|password|parola|giriş|oturum|credential|api.?key|secret/i.test(task)) return null;
+    if (!['act', 'dev'].includes(mode) || this._checkAbort(tabId)) return null;
     let session;
     try {
       const settings = await this._jevSettings();
-      if (!settings?.systemOneFastBrowser || !allowed.has('get_accessibility_tree') || !context.isCurrent()) return null;
+      if (!settings?.systemOneFastBrowser || !context.isCurrent()) return null;
+      let skipReason = '';
+      if (recovery) skipReason = 'recovery_turn';
+      else if (runOptions.cloudRun) skipReason = 'cloud_run';
+      else if (this.selectionGroundingScopes.has(tabId)) skipReason = 'selection_grounded';
+      else if (this._isStandaloneChatRun(runOptions)) skipReason = 'standalone_chat';
+      else if (jevVisualInputRequiresMainModel(messages)) skipReason = 'current_visual_input';
+      else if (/log.?in|sign.?in|password|passwd|passcode|\b(?:otp|token|secret)\b|one.?time.?code|parola|giriş|oturum|credential|api.?key/i.test(task)) skipReason = 'sensitive_task';
+      else if (!allowed.has('get_accessibility_tree')) skipReason = 'tool_policy';
+      if (skipReason) {
+        this.recordSystemOneVerdict(tabId, { decision: 'skip', reason: skipReason }, context);
+        return null;
+      }
       this._jevSessions ??= new Map();
       session = this._jevSessions.get(tabId);
       if (!session) { session = new JevFastSession(); this._jevSessions.set(tabId, session); }
@@ -6675,6 +6685,7 @@ export class Agent extends LoopDetector {
         this.recordSystemOneVerdict(tabId, { decision: 'fallback', reason }, context);
         return null;
       };
+      if (session.snapshot.hasSensitiveControls === true) return fallback('sensitive_controls');
       const taskText = String(task).slice(0, 4000);
       const cached = session.valueContext === this._jevValueContext(taskText, session.snapshot) ? session.values || [] : [];
       let request = buildJevBrowserRequest(taskText, session.snapshot, cached);