diff --git a/docs/architecture.md b/docs/architecture.md index 439831573..76523aefb 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -1042,7 +1042,8 @@ The existing AX walk supplies at most 24 structured controls with its own refs; page-authored ref strings are never parsed. Internal snapshots are stripped from public tool results and diagnostics. Identity, document, form structure, options, value and occlusion are checked again before dispatch. Frames, shadow roots, -credential/file fields and unsupported actions fall back. Field values come from +unsupported actions and pages containing credential, payment, OTP or file controls +fall back as a whole. Field values come from the active provider only after a confident fill action/target is selected. The first uncached fill uses a second Jev request to map those prepared values; clicks, completion candidates and fallbacks do not prepare text. Cached values @@ -1064,3 +1065,10 @@ run cancellation and model cost limits apply. `done` remains an active-model operation using existing evidence checks. RAG, skill routing and direct watch poll optimization are not part of this integration. See `test/jev/README.md` for benchmark protocol and its unverified live-performance status. + +Initial-page and automatic browser screenshots do not make the AX-only path +ineligible, and their pixels are never included in a Jev request. A current user +attachment, explicit screenshot-tool result or unknown non-text input routes that +decision to the active provider. The fast path may resume after that provider has +consumed the input. Trace exports render Jev routing, fallback and usage metadata, +including skip reasons, without exporting evidence. diff --git a/docs/privacy-and-data-flow.md b/docs/privacy-and-data-flow.md index 942e21d33..86084adfd 100644 --- a/docs/privacy-and-data-flow.md +++ b/docs/privacy-and-data-flow.md @@ -668,8 +668,13 @@ or scheduler settings do not enable them. Fast classification sends bounded request context; fast browser decisions send the task, up to 24 structured AX controls, observed options and bounded prepared field values. These may include ordinary personal text explicitly supplied for a form. Credential-related tasks -and credential/file controls are excluded from the fast path. Redaction remains -best effort. The active chat provider prepares free text and gives the final -answer; Jev receives neither screenshots nor full conversation history for -browser decisions. Separate requests use the same pinned model, cost accounting -and untrusted-data boundaries, with a one-second deadline and zero retries. +and pages containing credential, payment, OTP or file controls are excluded from +the fast path as a whole. Redaction remains best effort. Initial and automatic +browser screenshots do not disable the AX-only path, but their pixels are never +sent to Jev. A current user attachment, explicit screenshot-tool result or unknown +non-text input keeps that decision on the active chat provider. That provider also +prepares free text and gives the final answer; Jev receives neither screenshots nor +full conversation history for browser decisions. Separate requests use the same +pinned model, cost accounting and untrusted-data boundaries, with a one-second +deadline and zero retries. Exported traces show Jev decisions and skip reasons but +never the bounded request evidence. diff --git a/src/chrome/src/agent/agent.js b/src/chrome/src/agent/agent.js index 11881a763..15dd69542 100644 --- a/src/chrome/src/agent/agent.js +++ b/src/chrome/src/agent/agent.js @@ -1,4 +1,4 @@ -import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, JevFastSession } from './systemone-fast.js'; +import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js'; import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js'; import { createSystemOneJudge, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js'; import { SOCIAL_PLATFORMS, socialPublicationApiPlatform, normalizePublicationContract, publicationProgress, exactPublicationText, publicationMediaMatches, publicationContractMessages, publicationAuditMessages, publicationAuditAccepted } from './social-publish-contract.js'; @@ -6870,13 +6870,23 @@ export class Agent extends LoopDetector { async _maybeJevFastTurn(tabId, task, messages, mode, allowed, provider, costState, runOptions = {}, recovery = null) { const context = this.systemOneContext(tabId); - if (!['act', 'dev'].includes(mode) || recovery || runOptions.cloudRun || this.selectionGroundingScopes.has(tabId) || this._isStandaloneChatRun(runOptions) - || messages.some(message => Array.isArray(message?.content) && message.content.some(block => block?.type !== 'text')) - || this._checkAbort(tabId) || /log.?in|sign.?in|password|parola|giriş|oturum|credential|api.?key|secret/i.test(task)) return null; + if (!['act', 'dev'].includes(mode) || this._checkAbort(tabId)) return null; let session; try { const settings = await this._jevSettings(); - if (!settings?.systemOneFastBrowser || !allowed.has('get_accessibility_tree') || !context.isCurrent()) return null; + if (!settings?.systemOneFastBrowser || !context.isCurrent()) return null; + let skipReason = ''; + if (recovery) skipReason = 'recovery_turn'; + else if (runOptions.cloudRun) skipReason = 'cloud_run'; + else if (this.selectionGroundingScopes.has(tabId)) skipReason = 'selection_grounded'; + else if (this._isStandaloneChatRun(runOptions)) skipReason = 'standalone_chat'; + else if (jevVisualInputRequiresMainModel(messages)) skipReason = 'current_visual_input'; + else if (/log.?in|sign.?in|password|passwd|passcode|\b(?:otp|token|secret)\b|one.?time.?code|parola|giriş|oturum|credential|api.?key/i.test(task)) skipReason = 'sensitive_task'; + else if (!allowed.has('get_accessibility_tree')) skipReason = 'tool_policy'; + if (skipReason) { + this.recordSystemOneVerdict(tabId, { decision: 'skip', reason: skipReason }, context); + return null; + } this._jevSessions ??= new Map(); session = this._jevSessions.get(tabId); if (!session) { session = new JevFastSession(); this._jevSessions.set(tabId, session); } @@ -6891,6 +6901,7 @@ export class Agent extends LoopDetector { this.recordSystemOneVerdict(tabId, { decision: 'fallback', reason }, context); return null; }; + if (session.snapshot.hasSensitiveControls === true) return fallback('sensitive_controls'); const taskText = String(task).slice(0, 4000); const cached = session.valueContext === this._jevValueContext(taskText, session.snapshot) ? session.values || [] : []; let request = buildJevBrowserRequest(taskText, session.snapshot, cached); diff --git a/src/chrome/src/agent/systemone-fast.js b/src/chrome/src/agent/systemone-fast.js index c7ee70bd3..bfc8d43fd 100644 --- a/src/chrome/src/agent/systemone-fast.js +++ b/src/chrome/src/agent/systemone-fast.js @@ -2,6 +2,49 @@ import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js' export const JEV_CLASSIFIER_THRESHOLD = .85; export const JEV_BROWSER_THRESHOLD = .90; export const JEV_FAST_KEYS = ['systemOneEnabled', 'typesafeApiKey', 'systemOneFastClassifications', 'systemOneFastBrowser']; + +function messageText(message) { + if (typeof message?.content === 'string') return message.content; + if (!Array.isArray(message?.content)) return ''; + return message.content + .filter(block => block?.type === 'text' && typeof block.text === 'string') + .map(block => block.text) + .join('\n'); +} + +function hasNonTextBlock(message) { + return Array.isArray(message?.content) + && message.content.some(block => block?.type !== 'text'); +} + +// Only a visual input that the main model has not consumed yet blocks Jev. +// Initial and automatic browser captures are auxiliary context; Jev makes its +// decision from a fresh bounded AX snapshot and never receives those pixels. +// User attachments and explicit screenshot-tool results still require the main +// model for visual reasoning. Looking only after the latest assistant message +// lets Jev resume after that model has consumed an explicit visual observation. +export function jevVisualInputRequiresMainModel(messages) { + if (!Array.isArray(messages)) return false; + let start = 0; + for (let index = messages.length - 1; index >= 0; index -= 1) { + if (messages[index]?.role === 'assistant') { + start = index + 1; + break; + } + } + for (let index = start; index < messages.length; index += 1) { + const message = messages[index]; + if (!hasNonTextBlock(message)) continue; + const text = messageText(message); + if (text.includes('[UNTRUSTED USER ATTACHMENTS')) return true; + if (/\[UNTRUSTED SCREENSHOT[^\]]*Screenshot from your [^\]]+ call\./i.test(text)) return true; + if (/\[UNTRUSTED SCREENSHOT[^\]]*Capture ID:/i.test(text)) continue; + if (/\[UNTRUSTED CAPTURE[^\]]*Capture ID:/i.test(text)) continue; + return true; + } + return false; +} + export function confidentChoice(answer, threshold) { return answer?.type === 'choice' && typeof answer.confidence === 'number' && answer.confidence >= threshold && typeof answer.probabilities?.[answer.choice] === 'number' && answer.probabilities[answer.choice] >= threshold @@ -10,7 +53,7 @@ export function confidentChoice(answer, threshold) { const question = (instructions, criteria) => ({ type: 'choice', instructions, criteria }); const NONE = { none: 'No supported target; use the main model.' }; export function buildJevBrowserRequest(task, snapshot, values = []) { - if (!snapshot || !Array.isArray(snapshot.controls) || !snapshot.documentToken || !snapshot.structure) return null; + if (!snapshot || snapshot.hasSensitiveControls === true || !Array.isArray(snapshot.controls) || !snapshot.documentToken || !snapshot.structure) return null; const controls = snapshot.controls.slice(0, 24); const targets = kind => Object.fromEntries(controls.filter(c => c.kinds.includes(kind)).map(c => [c.ref, `Observed ${kind} target ${c.ref} in state.controls.`])); const choices = { click: 'Click a visible control or link, including a requested final submit/save/send.', fill: 'Fill fields required by the user task. Values can be prepared after this action is selected.', select: 'Select an observed native option.', check: 'Set a checkbox state.', scroll_down: 'Scroll down to reveal controls.', scroll_up: 'Scroll up.', wait: 'Wait for the page to settle.', done: 'Candidate completion: ask the main model to verify evidence and respond.', fallback: 'Unsupported, ambiguous, visual, iframe, shadow, upload, keyboard, code or WebMCP work: use the main model.' }; @@ -101,7 +144,7 @@ export class JevFastSession { get fallbackBlocked() { return this.fallbackCount >= 2; } recordFallback() { this.fallbackCount++; this.queue = []; } observe(snapshot) { - const context = JSON.stringify([snapshot?.documentToken, snapshot?.pageUrl, snapshot?.structure, snapshot?.progress]); + const context = JSON.stringify([snapshot?.documentToken, snapshot?.pageUrl, snapshot?.structure, snapshot?.progress, snapshot?.hasSensitiveControls === true]); if (context !== this.fallbackContext) { this.fallbackCount = 0; this.completionCandidate = false; @@ -111,7 +154,8 @@ export class JevFastSession { else if (this.pending) this.noProgress = 0; this.pending = false; if (this.noProgress >= 2) this.disabled = true; - if (this.snapshot?.structure !== snapshot?.structure || this.snapshot?.documentToken !== snapshot?.documentToken) this.queue = []; + if (this.snapshot?.structure !== snapshot?.structure || this.snapshot?.documentToken !== snapshot?.documentToken + || (this.snapshot?.hasSensitiveControls === true) !== (snapshot?.hasSensitiveControls === true)) this.queue = []; this.snapshot = snapshot; } dispatched(result) { diff --git a/src/chrome/src/agent/trace-export.js b/src/chrome/src/agent/trace-export.js index f072cbc7f..95b945457 100644 --- a/src/chrome/src/agent/trace-export.js +++ b/src/chrome/src/agent/trace-export.js @@ -352,6 +352,21 @@ export function tracesToMarkdown(runsWithEvents, { const details = [oneLine(d.context), oneLine(d.visionRoute), oneLine(d.model), oneLine(d.captureId)] .filter(Boolean).join(' · '); md += `- 👁 Vision route${details ? `: ${details}` : ''}${d.fallbackReason ? ` · fallback=${oneLine(d.fallbackReason)}` : ''}\n`; + } else if (ev.kind === 'note' && d.note === 'system_one') { + const metadata = d.extra || {}; + const usage = metadata.usage || {}; + const inputTokens = Number(usage.prompt_tokens ?? usage.input_tokens); + const outputTokens = Number(usage.completion_tokens ?? usage.output_tokens); + const details = [ + metadata.reason ? `reason=${oneLine(metadata.reason)}` : '', + metadata.model ? `model=${oneLine(metadata.model)}` : '', + Number.isFinite(metadata.latencyMs) ? `${metadata.latencyMs} ms` : '', + Number.isFinite(metadata.estimatedCostUsd) ? `$${metadata.estimatedCostUsd.toFixed(6)}` : '', + Number.isFinite(inputTokens) || Number.isFinite(outputTokens) + ? `${Number.isFinite(inputTokens) ? inputTokens : 0} in / ${Number.isFinite(outputTokens) ? outputTokens : 0} out` + : '', + ].filter(Boolean).join(' · '); + md += `- ⚡ Jev: ${oneLine(metadata.decision || 'event')}${details ? ` · ${details}` : ''}\n`; } else if (ev.kind === 'note' && d.note === 'vision_status') { const status = d.extra || {}; const progress = Number.isFinite(Number(status.progress)) ? `${Math.round(Number(status.progress))}%` : ''; diff --git a/src/chrome/src/content/accessibility-tree.js b/src/chrome/src/content/accessibility-tree.js index 61ba62c56..c85f30c57 100644 --- a/src/chrome/src/content/accessibility-tree.js +++ b/src/chrome/src/content/accessibility-tree.js @@ -715,12 +715,18 @@ // parses page-authored ref strings and cannot target frames or shadow roots. let jevCollector = null; let lastJevSnapshot = null; + let jevSensitiveControlSeen = false; function jevControl(el) { if (window.top !== window || el.getRootNode() !== document || !isInteractive(el)) return null; const tag = el.tagName.toLowerCase(); const type = String(el.type || '').toLowerCase(); const name = String(getAccessibleName(el) || '').slice(0, 120); - if (['password', 'file', 'hidden'].includes(type) || /password|passwd|api.?key|token|secret|credit.?card|cvv|cvc/i.test([name, el.name, el.id, el.autocomplete].join(' '))) return null; + const identity = [name, el.name, el.id, el.autocomplete].join(' '); + if (['password', 'file', 'hidden'].includes(type) + || /\b(?:password|passwd|passcode|pin|otp|token|secret|cvv|cvc|csc|ssn|iban)\b|\bcc-(?:name|given-name|additional-name|family-name|number|exp|exp-month|exp-year|csc|type)\b|api.?key|one[-_\s]?time(?:[-_\s]?code)?|verification.?code|security.?code|auth(?:entication)?.?code|credit.?card|card.?number|social.?security|(?:routing|account).?number/i.test(identity)) { + jevSensitiveControlSeen = true; + return null; + } const role = getRole(el); const kinds = []; if (tag === 'select') kinds.push('select'); @@ -741,15 +747,18 @@ return control; } function collectJevControl(el) { - if (!jevCollector || jevCollector.size >= 24) return; - try { const control = jevControl(el); if (control) jevCollector.set(control.ref, control); } catch { /* Optional inventory must not break the normal AX reader. */ } + if (!jevCollector) return; + try { + const control = jevControl(el); + if (control && jevCollector.size < 24) jevCollector.set(control.ref, control); + } catch { /* Optional inventory must not break the normal AX reader. */ } } function jevSnapshot() { return lastJevSnapshot; } function jevValidate(binding) { if (!binding || binding.pageUrl !== location.href) return false; generateAccessibilityTree('interactive', 10, 3500); const snapshot = lastJevSnapshot; - if (!snapshot || snapshot.structure !== binding.structure) return false; + if (!snapshot || snapshot.hasSensitiveControls === true || snapshot.structure !== binding.structure) return false; const target = snapshot.controls.find(c => c.ref === binding.ref); const el = lookup(binding.ref); if (!target || target.disabled || target.signature !== binding.signature || !el || el.getRootNode() !== document) return false; @@ -1452,6 +1461,7 @@ function generateAccessibilityTree(filter, maxDepth, maxChars, refId, page, expectedTreeRevision) { jevCollector = filter === 'interactive' && !refId && (!page || page === 1) ? new Map() : null; lastJevSnapshot = null; + jevSensitiveControlSeen = false; try { ensureRefScope(); const effFilter = filter || 'all'; @@ -1723,7 +1733,7 @@ if (jevCollector) { const controls = [...jevCollector.values()]; const structure = fingerprintTreeContent(JSON.stringify(controls.map(({ value, checked, signature, ...control }) => control))); - lastJevSnapshot = { controls, structure, progress: fingerprintTreeContent(JSON.stringify(controls)) + ':' + Math.round(scrollY) }; + lastJevSnapshot = { controls, structure, progress: fingerprintTreeContent(JSON.stringify(controls)) + ':' + Math.round(scrollY), hasSensitiveControls: jevSensitiveControlSeen }; } jevCollector = null; } diff --git a/src/firefox/src/agent/agent.js b/src/firefox/src/agent/agent.js index 5787a0c3e..2629f81bc 100644 --- a/src/firefox/src/agent/agent.js +++ b/src/firefox/src/agent/agent.js @@ -1,4 +1,4 @@ -import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, JevFastSession } from './systemone-fast.js'; +import { JEV_FAST_KEYS, JEV_CLASSIFIER_THRESHOLD, JEV_BROWSER_THRESHOLD, confidentChoice, buildJevBrowserRequest, decideJevBrowser, jevVisualInputRequiresMainModel, JevFastSession } from './systemone-fast.js'; import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js'; import { createSystemOneJudge, SYSTEM_ONE_COST_PROVIDER } from './systemone-judge.js'; import { firefoxBidi } from '../bidi/client.js'; @@ -6654,13 +6654,23 @@ export class Agent extends LoopDetector { async _maybeJevFastTurn(tabId, task, messages, mode, allowed, provider, costState, runOptions = {}, recovery = null) { const context = this.systemOneContext(tabId); - if (!['act', 'dev'].includes(mode) || recovery || runOptions.cloudRun || this.selectionGroundingScopes.has(tabId) || this._isStandaloneChatRun(runOptions) - || messages.some(message => Array.isArray(message?.content) && message.content.some(block => block?.type !== 'text')) - || this._checkAbort(tabId) || /log.?in|sign.?in|password|parola|giriş|oturum|credential|api.?key|secret/i.test(task)) return null; + if (!['act', 'dev'].includes(mode) || this._checkAbort(tabId)) return null; let session; try { const settings = await this._jevSettings(); - if (!settings?.systemOneFastBrowser || !allowed.has('get_accessibility_tree') || !context.isCurrent()) return null; + if (!settings?.systemOneFastBrowser || !context.isCurrent()) return null; + let skipReason = ''; + if (recovery) skipReason = 'recovery_turn'; + else if (runOptions.cloudRun) skipReason = 'cloud_run'; + else if (this.selectionGroundingScopes.has(tabId)) skipReason = 'selection_grounded'; + else if (this._isStandaloneChatRun(runOptions)) skipReason = 'standalone_chat'; + else if (jevVisualInputRequiresMainModel(messages)) skipReason = 'current_visual_input'; + else if (/log.?in|sign.?in|password|passwd|passcode|\b(?:otp|token|secret)\b|one.?time.?code|parola|giriş|oturum|credential|api.?key/i.test(task)) skipReason = 'sensitive_task'; + else if (!allowed.has('get_accessibility_tree')) skipReason = 'tool_policy'; + if (skipReason) { + this.recordSystemOneVerdict(tabId, { decision: 'skip', reason: skipReason }, context); + return null; + } this._jevSessions ??= new Map(); session = this._jevSessions.get(tabId); if (!session) { session = new JevFastSession(); this._jevSessions.set(tabId, session); } @@ -6675,6 +6685,7 @@ export class Agent extends LoopDetector { this.recordSystemOneVerdict(tabId, { decision: 'fallback', reason }, context); return null; }; + if (session.snapshot.hasSensitiveControls === true) return fallback('sensitive_controls'); const taskText = String(task).slice(0, 4000); const cached = session.valueContext === this._jevValueContext(taskText, session.snapshot) ? session.values || [] : []; let request = buildJevBrowserRequest(taskText, session.snapshot, cached); diff --git a/src/firefox/src/agent/systemone-fast.js b/src/firefox/src/agent/systemone-fast.js index c7ee70bd3..bfc8d43fd 100644 --- a/src/firefox/src/agent/systemone-fast.js +++ b/src/firefox/src/agent/systemone-fast.js @@ -2,6 +2,49 @@ import { redactSystemOneText, wrapSystemOneData } from './systemone-evidence.js' export const JEV_CLASSIFIER_THRESHOLD = .85; export const JEV_BROWSER_THRESHOLD = .90; export const JEV_FAST_KEYS = ['systemOneEnabled', 'typesafeApiKey', 'systemOneFastClassifications', 'systemOneFastBrowser']; + +function messageText(message) { + if (typeof message?.content === 'string') return message.content; + if (!Array.isArray(message?.content)) return ''; + return message.content + .filter(block => block?.type === 'text' && typeof block.text === 'string') + .map(block => block.text) + .join('\n'); +} + +function hasNonTextBlock(message) { + return Array.isArray(message?.content) + && message.content.some(block => block?.type !== 'text'); +} + +// Only a visual input that the main model has not consumed yet blocks Jev. +// Initial and automatic browser captures are auxiliary context; Jev makes its +// decision from a fresh bounded AX snapshot and never receives those pixels. +// User attachments and explicit screenshot-tool results still require the main +// model for visual reasoning. Looking only after the latest assistant message +// lets Jev resume after that model has consumed an explicit visual observation. +export function jevVisualInputRequiresMainModel(messages) { + if (!Array.isArray(messages)) return false; + let start = 0; + for (let index = messages.length - 1; index >= 0; index -= 1) { + if (messages[index]?.role === 'assistant') { + start = index + 1; + break; + } + } + for (let index = start; index < messages.length; index += 1) { + const message = messages[index]; + if (!hasNonTextBlock(message)) continue; + const text = messageText(message); + if (text.includes('[UNTRUSTED USER ATTACHMENTS')) return true; + if (/\[UNTRUSTED SCREENSHOT[^\]]*Screenshot from your [^\]]+ call\./i.test(text)) return true; + if (/\[UNTRUSTED SCREENSHOT[^\]]*Capture ID:/i.test(text)) continue; + if (/\[UNTRUSTED CAPTURE[^\]]*Capture ID:/i.test(text)) continue; + return true; + } + return false; +} + export function confidentChoice(answer, threshold) { return answer?.type === 'choice' && typeof answer.confidence === 'number' && answer.confidence >= threshold && typeof answer.probabilities?.[answer.choice] === 'number' && answer.probabilities[answer.choice] >= threshold @@ -10,7 +53,7 @@ export function confidentChoice(answer, threshold) { const question = (instructions, criteria) => ({ type: 'choice', instructions, criteria }); const NONE = { none: 'No supported target; use the main model.' }; export function buildJevBrowserRequest(task, snapshot, values = []) { - if (!snapshot || !Array.isArray(snapshot.controls) || !snapshot.documentToken || !snapshot.structure) return null; + if (!snapshot || snapshot.hasSensitiveControls === true || !Array.isArray(snapshot.controls) || !snapshot.documentToken || !snapshot.structure) return null; const controls = snapshot.controls.slice(0, 24); const targets = kind => Object.fromEntries(controls.filter(c => c.kinds.includes(kind)).map(c => [c.ref, `Observed ${kind} target ${c.ref} in state.controls.`])); const choices = { click: 'Click a visible control or link, including a requested final submit/save/send.', fill: 'Fill fields required by the user task. Values can be prepared after this action is selected.', select: 'Select an observed native option.', check: 'Set a checkbox state.', scroll_down: 'Scroll down to reveal controls.', scroll_up: 'Scroll up.', wait: 'Wait for the page to settle.', done: 'Candidate completion: ask the main model to verify evidence and respond.', fallback: 'Unsupported, ambiguous, visual, iframe, shadow, upload, keyboard, code or WebMCP work: use the main model.' }; @@ -101,7 +144,7 @@ export class JevFastSession { get fallbackBlocked() { return this.fallbackCount >= 2; } recordFallback() { this.fallbackCount++; this.queue = []; } observe(snapshot) { - const context = JSON.stringify([snapshot?.documentToken, snapshot?.pageUrl, snapshot?.structure, snapshot?.progress]); + const context = JSON.stringify([snapshot?.documentToken, snapshot?.pageUrl, snapshot?.structure, snapshot?.progress, snapshot?.hasSensitiveControls === true]); if (context !== this.fallbackContext) { this.fallbackCount = 0; this.completionCandidate = false; @@ -111,7 +154,8 @@ export class JevFastSession { else if (this.pending) this.noProgress = 0; this.pending = false; if (this.noProgress >= 2) this.disabled = true; - if (this.snapshot?.structure !== snapshot?.structure || this.snapshot?.documentToken !== snapshot?.documentToken) this.queue = []; + if (this.snapshot?.structure !== snapshot?.structure || this.snapshot?.documentToken !== snapshot?.documentToken + || (this.snapshot?.hasSensitiveControls === true) !== (snapshot?.hasSensitiveControls === true)) this.queue = []; this.snapshot = snapshot; } dispatched(result) { diff --git a/src/firefox/src/agent/trace-export.js b/src/firefox/src/agent/trace-export.js index 2a1efcaa2..246beee2f 100644 --- a/src/firefox/src/agent/trace-export.js +++ b/src/firefox/src/agent/trace-export.js @@ -358,6 +358,21 @@ export function tracesToMarkdown(runsWithEvents, { const details = [oneLine(d.context), oneLine(d.visionRoute), oneLine(d.model), oneLine(d.captureId)] .filter(Boolean).join(' · '); md += `- 👁 Vision route${details ? `: ${details}` : ''}${d.fallbackReason ? ` · fallback=${oneLine(d.fallbackReason)}` : ''}\n`; + } else if (ev.kind === 'note' && d.note === 'system_one') { + const metadata = d.extra || {}; + const usage = metadata.usage || {}; + const inputTokens = Number(usage.prompt_tokens ?? usage.input_tokens); + const outputTokens = Number(usage.completion_tokens ?? usage.output_tokens); + const details = [ + metadata.reason ? `reason=${oneLine(metadata.reason)}` : '', + metadata.model ? `model=${oneLine(metadata.model)}` : '', + Number.isFinite(metadata.latencyMs) ? `${metadata.latencyMs} ms` : '', + Number.isFinite(metadata.estimatedCostUsd) ? `$${metadata.estimatedCostUsd.toFixed(6)}` : '', + Number.isFinite(inputTokens) || Number.isFinite(outputTokens) + ? `${Number.isFinite(inputTokens) ? inputTokens : 0} in / ${Number.isFinite(outputTokens) ? outputTokens : 0} out` + : '', + ].filter(Boolean).join(' · '); + md += `- ⚡ Jev: ${oneLine(metadata.decision || 'event')}${details ? ` · ${details}` : ''}\n`; } else if (ev.kind === 'note' && d.note === 'vision_status') { const status = d.extra || {}; const progress = Number.isFinite(Number(status.progress)) ? `${Math.round(Number(status.progress))}%` : ''; diff --git a/src/firefox/src/content/accessibility-tree.js b/src/firefox/src/content/accessibility-tree.js index 61ba62c56..c85f30c57 100644 --- a/src/firefox/src/content/accessibility-tree.js +++ b/src/firefox/src/content/accessibility-tree.js @@ -715,12 +715,18 @@ // parses page-authored ref strings and cannot target frames or shadow roots. let jevCollector = null; let lastJevSnapshot = null; + let jevSensitiveControlSeen = false; function jevControl(el) { if (window.top !== window || el.getRootNode() !== document || !isInteractive(el)) return null; const tag = el.tagName.toLowerCase(); const type = String(el.type || '').toLowerCase(); const name = String(getAccessibleName(el) || '').slice(0, 120); - if (['password', 'file', 'hidden'].includes(type) || /password|passwd|api.?key|token|secret|credit.?card|cvv|cvc/i.test([name, el.name, el.id, el.autocomplete].join(' '))) return null; + const identity = [name, el.name, el.id, el.autocomplete].join(' '); + if (['password', 'file', 'hidden'].includes(type) + || /\b(?:password|passwd|passcode|pin|otp|token|secret|cvv|cvc|csc|ssn|iban)\b|\bcc-(?:name|given-name|additional-name|family-name|number|exp|exp-month|exp-year|csc|type)\b|api.?key|one[-_\s]?time(?:[-_\s]?code)?|verification.?code|security.?code|auth(?:entication)?.?code|credit.?card|card.?number|social.?security|(?:routing|account).?number/i.test(identity)) { + jevSensitiveControlSeen = true; + return null; + } const role = getRole(el); const kinds = []; if (tag === 'select') kinds.push('select'); @@ -741,15 +747,18 @@ return control; } function collectJevControl(el) { - if (!jevCollector || jevCollector.size >= 24) return; - try { const control = jevControl(el); if (control) jevCollector.set(control.ref, control); } catch { /* Optional inventory must not break the normal AX reader. */ } + if (!jevCollector) return; + try { + const control = jevControl(el); + if (control && jevCollector.size < 24) jevCollector.set(control.ref, control); + } catch { /* Optional inventory must not break the normal AX reader. */ } } function jevSnapshot() { return lastJevSnapshot; } function jevValidate(binding) { if (!binding || binding.pageUrl !== location.href) return false; generateAccessibilityTree('interactive', 10, 3500); const snapshot = lastJevSnapshot; - if (!snapshot || snapshot.structure !== binding.structure) return false; + if (!snapshot || snapshot.hasSensitiveControls === true || snapshot.structure !== binding.structure) return false; const target = snapshot.controls.find(c => c.ref === binding.ref); const el = lookup(binding.ref); if (!target || target.disabled || target.signature !== binding.signature || !el || el.getRootNode() !== document) return false; @@ -1452,6 +1461,7 @@ function generateAccessibilityTree(filter, maxDepth, maxChars, refId, page, expectedTreeRevision) { jevCollector = filter === 'interactive' && !refId && (!page || page === 1) ? new Map() : null; lastJevSnapshot = null; + jevSensitiveControlSeen = false; try { ensureRefScope(); const effFilter = filter || 'all'; @@ -1723,7 +1733,7 @@ if (jevCollector) { const controls = [...jevCollector.values()]; const structure = fingerprintTreeContent(JSON.stringify(controls.map(({ value, checked, signature, ...control }) => control))); - lastJevSnapshot = { controls, structure, progress: fingerprintTreeContent(JSON.stringify(controls)) + ':' + Math.round(scrollY) }; + lastJevSnapshot = { controls, structure, progress: fingerprintTreeContent(JSON.stringify(controls)) + ':' + Math.round(scrollY), hasSensitiveControls: jevSensitiveControlSeen }; } jevCollector = null; } diff --git a/test/jev/README.md b/test/jev/README.md index b611eb6eb..33b800d95 100644 --- a/test/jev/README.md +++ b/test/jev/README.md @@ -37,6 +37,14 @@ cost. Acceptance requires all 120 attempts, at least 25% lower median duration, no success regression and no wrong or repeated Jev actions. No general speed claim should be made from fixture-only or mocked results. +For manual runs, exported traces now contain `⚡ Jev` entries for routing, +fallback and usage. An initial or automatic browser screenshot does not by itself +disable the AX-only path. `reason=current_visual_input` means a current user +attachment, explicit screenshot-tool result or unknown non-text input kept that +decision on the active provider. Sensitive controls use +`reason=sensitive_controls`. No screenshot pixels or bounded Jev evidence are +included in the export. + ## Settings and documentation checks Configure Jev under **Settings → Assistive Models → Jev (TypeSafe)**, after diff --git a/test/run.js b/test/run.js index 67d605f57..ff889602a 100644 --- a/test/run.js +++ b/test/run.js @@ -10995,6 +10995,25 @@ test('trace export: proves visual delivery without exporting pixels or OCR text' } }); +test('trace export: renders Jev routing and usage without private evidence', () => { + const runs = [{ + run: { runId: 'jev-routing', userMessage: 'Save this record', status: 'done' }, + events: [ + { kind: 'note', data: { note: 'system_one', extra: { decision: 'skip', reason: 'current_visual_input' } } }, + { kind: 'note', data: { note: 'system_one', extra: { + decision: 'usage', model: 'jev-1.13.0', latencyMs: 25, estimatedCostUsd: 0.001, + usage: { prompt_tokens: 12, completion_tokens: 2 }, evidence: 'PRIVATE_JEV_EVIDENCE', + } } }, + ], + }]; + for (const [label, serialize] of [['chrome', tracesToMarkdown], ['firefox', tracesToMarkdownFx]]) { + const { markdown } = serialize(runs); + assert.match(markdown, /⚡ Jev: skip · reason=current_visual_input/, `${label}: Jev skip reason missing`); + assert.match(markdown, /⚡ Jev: usage · model=jev-1\.13\.0 · 25 ms · \$0\.001000 · 12 in \/ 2 out/, `${label}: Jev usage missing`); + assert.doesNotMatch(markdown, /PRIVATE_JEV_EVIDENCE/, `${label}: Jev evidence leaked`); + } +}); + test('trace export: reports prompt/runtime alignment without fingerprinting private content', () => { const runtimeContext = buildTrustedRuntimeContextCh({ now: new Date('2026-08-11T10:00:00.000Z'), diff --git a/test/systemone-fast-dom.mjs b/test/systemone-fast-dom.mjs index ef953500c..70cec6dea 100644 --- a/test/systemone-fast-dom.mjs +++ b/test/systemone-fast-dom.mjs @@ -6,7 +6,7 @@ const fixture = `