Tara Menon
d5cab65099
The daemon JS files (daemon.js, server.js, browser.js, protocol.js, snapshot.js, ipc.js) are now EMBEDDED in the compiled Rust binary via include_dir! and extracted to ~/.gyandoot/daemon/ on first run. This eliminates the need to npm install -g camoufox-browser@0.1.0, which bundles an incompatible camoufox-js@0.9.3 that caused: - 'Cannot open database' errors - Re-downloading 298MB browser on every navigation - Circular install/remove/install loops Now the daemon imports camoufox-js from the global npm install (v0.11.0) which is installed by install.sh. Clean, no bundled deps, no conflicts. Also removed: is_npm_package_installed(), install_npm_package(), find_daemon_js() — all replaced by gyandoot_browser::daemon::extract_daemon()
386 lines
12 KiB
JavaScript
386 lines
12 KiB
JavaScript
/**
|
|
* Enhanced snapshot with element refs for deterministic element selection.
|
|
*
|
|
* This is largely compatible with agent-browser's snapshot/ref workflow:
|
|
* - `snapshot -i` returns a tree with `[ref=e1]` markers.
|
|
* - Subsequent actions can target `@e1` deterministically.
|
|
*/
|
|
let refCounter = 0;
|
|
export function resetRefs() {
|
|
refCounter = 0;
|
|
}
|
|
function nextRef() {
|
|
return `e${++refCounter}`;
|
|
}
|
|
const INTERACTIVE_ROLES = new Set([
|
|
"button",
|
|
"link",
|
|
"textbox",
|
|
"checkbox",
|
|
"radio",
|
|
"combobox",
|
|
"listbox",
|
|
"menuitem",
|
|
"menuitemcheckbox",
|
|
"menuitemradio",
|
|
"option",
|
|
"searchbox",
|
|
"slider",
|
|
"spinbutton",
|
|
"switch",
|
|
"tab",
|
|
"treeitem"
|
|
]);
|
|
const CONTENT_ROLES = new Set([
|
|
"heading",
|
|
"cell",
|
|
"gridcell",
|
|
"columnheader",
|
|
"rowheader",
|
|
"listitem",
|
|
"article",
|
|
"region",
|
|
"main",
|
|
"navigation"
|
|
]);
|
|
const STRUCTURAL_ROLES = new Set([
|
|
"generic",
|
|
"group",
|
|
"list",
|
|
"table",
|
|
"row",
|
|
"rowgroup",
|
|
"grid",
|
|
"treegrid",
|
|
"menu",
|
|
"menubar",
|
|
"toolbar",
|
|
"tablist",
|
|
"tree",
|
|
"directory",
|
|
"document",
|
|
"application",
|
|
"presentation",
|
|
"none"
|
|
]);
|
|
function buildSelector(role, name) {
|
|
const escapedName = JSON.stringify(name);
|
|
return `getByRole('${role}', { name: ${escapedName}, exact: true })`;
|
|
}
|
|
async function findCursorInteractiveElements(page, selector) {
|
|
const rootSelector = selector || "body";
|
|
const scriptBody = `(rootSel) => {
|
|
const results = [];
|
|
|
|
const interactiveRoles = new Set([
|
|
'button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', 'listbox',
|
|
'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'searchbox',
|
|
'slider', 'spinbutton', 'switch', 'tab', 'treeitem'
|
|
]);
|
|
|
|
const interactiveTags = new Set([
|
|
'a', 'button', 'input', 'select', 'textarea', 'details', 'summary'
|
|
]);
|
|
|
|
const root = document.querySelector(rootSel) || document.body;
|
|
const allElements = root.querySelectorAll('*');
|
|
|
|
const buildSelector = (el) => {
|
|
const testId = el.getAttribute('data-testid');
|
|
if (testId) return '[data-testid=\"' + testId + '\"]';
|
|
if (el.id) return '#' + CSS.escape(el.id);
|
|
|
|
const path = [];
|
|
let current = el;
|
|
while (current && current !== document.body) {
|
|
let sel = current.tagName.toLowerCase();
|
|
const classes = Array.from(current.classList).filter(c => c.trim());
|
|
if (classes.length > 0) sel += '.' + CSS.escape(classes[0]);
|
|
|
|
const parent = current.parentElement;
|
|
if (parent) {
|
|
const siblings = Array.from(parent.children);
|
|
const matching = siblings.filter(s => {
|
|
if (s.tagName !== current.tagName) return false;
|
|
if (classes.length > 0 && !s.classList.contains(classes[0])) return false;
|
|
return true;
|
|
});
|
|
if (matching.length > 1) {
|
|
const idx = matching.indexOf(current) + 1;
|
|
sel += ':nth-of-type(' + idx + ')';
|
|
}
|
|
}
|
|
path.unshift(sel);
|
|
current = current.parentElement;
|
|
if (path.length >= 1) {
|
|
try {
|
|
const candidate = path.join(' > ');
|
|
if (document.querySelectorAll(candidate).length === 1) break;
|
|
} catch (e) {}
|
|
}
|
|
if (path.length >= 10) break;
|
|
}
|
|
return path.join(' > ');
|
|
};
|
|
|
|
for (const el of allElements) {
|
|
const tagName = el.tagName.toLowerCase();
|
|
if (interactiveTags.has(tagName)) continue;
|
|
|
|
const role = el.getAttribute('role');
|
|
if (role && interactiveRoles.has(role.toLowerCase())) continue;
|
|
|
|
const computedStyle = getComputedStyle(el);
|
|
const hasCursorPointer = computedStyle.cursor === 'pointer';
|
|
const hasOnClick = el.hasAttribute('onclick') || el.onclick !== null;
|
|
const tabIndex = el.getAttribute('tabindex');
|
|
const hasTabIndex = tabIndex !== null && tabIndex !== '-1';
|
|
|
|
if (!hasCursorPointer && !hasOnClick && !hasTabIndex) continue;
|
|
|
|
if (hasCursorPointer && !hasOnClick && !hasTabIndex) {
|
|
const parent = el.parentElement;
|
|
if (parent && getComputedStyle(parent).cursor === 'pointer') continue;
|
|
}
|
|
|
|
const text = (el.textContent || '').trim().slice(0, 100);
|
|
if (!text) continue;
|
|
|
|
const rect = el.getBoundingClientRect();
|
|
if (rect.width === 0 || rect.height === 0) continue;
|
|
|
|
results.push({
|
|
selector: buildSelector(el),
|
|
text,
|
|
tagName,
|
|
hasOnClick,
|
|
hasCursorPointer,
|
|
hasTabIndex
|
|
});
|
|
}
|
|
return results;
|
|
}`;
|
|
// eslint-disable-next-line @typescript-eslint/no-implied-eval
|
|
const fn = new Function("return " + scriptBody)();
|
|
return page.evaluate(fn, rootSelector);
|
|
}
|
|
export async function getEnhancedSnapshot(page, options = {}) {
|
|
resetRefs();
|
|
const refs = {};
|
|
const locator = options.selector ? page.locator(options.selector) : page.locator(":root");
|
|
const ariaTree = await locator.ariaSnapshot();
|
|
if (!ariaTree) {
|
|
return { tree: "(empty)", refs: {} };
|
|
}
|
|
const enhancedTree = processAriaTree(ariaTree, refs, options);
|
|
if (options.cursor) {
|
|
const cursorElements = await findCursorInteractiveElements(page, options.selector);
|
|
const existingTexts = new Set(Object.values(refs).map((r) => r.name.toLowerCase()));
|
|
for (const m of enhancedTree.matchAll(/"([^"]+)"/g)) {
|
|
existingTexts.add(m[1].toLowerCase());
|
|
}
|
|
const additionalLines = [];
|
|
for (const el of cursorElements) {
|
|
const elTextLower = el.text.toLowerCase();
|
|
if (existingTexts.has(elTextLower))
|
|
continue;
|
|
existingTexts.add(elTextLower);
|
|
const ref = nextRef();
|
|
const role = el.hasCursorPointer || el.hasOnClick ? "clickable" : "focusable";
|
|
refs[ref] = { selector: el.selector, role, name: el.text };
|
|
const hints = [];
|
|
if (el.hasCursorPointer)
|
|
hints.push("cursor:pointer");
|
|
if (el.hasOnClick)
|
|
hints.push("onclick");
|
|
if (el.hasTabIndex)
|
|
hints.push("tabindex");
|
|
additionalLines.push(`- ${role} "${el.text}" [ref=${ref}] [${hints.join(", ")}]`);
|
|
}
|
|
if (additionalLines.length > 0) {
|
|
const separator = enhancedTree === "(no interactive elements)"
|
|
? ""
|
|
: "\n# Cursor-interactive elements:\n";
|
|
const base = enhancedTree === "(no interactive elements)" ? "" : enhancedTree;
|
|
return { tree: base + separator + additionalLines.join("\n"), refs };
|
|
}
|
|
}
|
|
return { tree: enhancedTree, refs };
|
|
}
|
|
function createRoleNameTracker() {
|
|
const counts = new Map();
|
|
const refsByKey = new Map();
|
|
return {
|
|
counts,
|
|
refsByKey,
|
|
getKey(role, name) {
|
|
return `${role}:${name ?? ""}`;
|
|
},
|
|
getNextIndex(role, name) {
|
|
const key = this.getKey(role, name);
|
|
const current = counts.get(key) ?? 0;
|
|
counts.set(key, current + 1);
|
|
return current;
|
|
},
|
|
trackRef(role, name, ref) {
|
|
const key = this.getKey(role, name);
|
|
const list = refsByKey.get(key) ?? [];
|
|
list.push(ref);
|
|
refsByKey.set(key, list);
|
|
},
|
|
getDuplicateKeys() {
|
|
const duplicates = new Set();
|
|
for (const [key, list] of refsByKey) {
|
|
if (list.length > 1)
|
|
duplicates.add(key);
|
|
}
|
|
return duplicates;
|
|
}
|
|
};
|
|
}
|
|
function processAriaTree(ariaTree, refs, options) {
|
|
const lines = ariaTree.split("\n");
|
|
const result = [];
|
|
const tracker = createRoleNameTracker();
|
|
if (options.interactive) {
|
|
for (const line of lines) {
|
|
const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
|
|
if (!match)
|
|
continue;
|
|
const [, , role, name, suffix] = match;
|
|
const roleLower = role.toLowerCase();
|
|
if (INTERACTIVE_ROLES.has(roleLower)) {
|
|
const ref = nextRef();
|
|
const resolvedName = name ?? "";
|
|
const nth = tracker.getNextIndex(roleLower, resolvedName);
|
|
tracker.trackRef(roleLower, resolvedName, ref);
|
|
refs[ref] = {
|
|
selector: buildSelector(roleLower, resolvedName),
|
|
role: roleLower,
|
|
name: resolvedName,
|
|
nth
|
|
};
|
|
let enhanced = `- ${role}`;
|
|
if (name)
|
|
enhanced += ` "${name}"`;
|
|
enhanced += ` [ref=${ref}]`;
|
|
if (nth > 0)
|
|
enhanced += ` [nth=${nth}]`;
|
|
if (suffix && suffix.includes("["))
|
|
enhanced += suffix;
|
|
result.push(enhanced);
|
|
}
|
|
}
|
|
removeNthFromNonDuplicates(refs, tracker);
|
|
return result.join("\n") || "(no interactive elements)";
|
|
}
|
|
for (const line of lines) {
|
|
const processed = processLine(line, refs, options, tracker);
|
|
if (processed !== null)
|
|
result.push(processed);
|
|
}
|
|
removeNthFromNonDuplicates(refs, tracker);
|
|
if (options.compact) {
|
|
return compactTree(result.join("\n"));
|
|
}
|
|
return result.join("\n");
|
|
}
|
|
function removeNthFromNonDuplicates(refs, tracker) {
|
|
const duplicateKeys = tracker.getDuplicateKeys();
|
|
for (const [ref, data] of Object.entries(refs)) {
|
|
const key = tracker.getKey(data.role, data.name);
|
|
if (!duplicateKeys.has(key)) {
|
|
delete refs[ref].nth;
|
|
}
|
|
}
|
|
}
|
|
function getIndentLevel(line) {
|
|
const match = line.match(/^(\s*)/);
|
|
return match ? Math.floor(match[1].length / 2) : 0;
|
|
}
|
|
function processLine(line, refs, options, tracker) {
|
|
const depth = getIndentLevel(line);
|
|
if (options.maxDepth !== undefined && depth > options.maxDepth) {
|
|
return null;
|
|
}
|
|
const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
|
|
if (!match) {
|
|
if (options.interactive)
|
|
return null;
|
|
return line;
|
|
}
|
|
const [, prefix, role, name, suffix] = match;
|
|
const roleLower = role.toLowerCase();
|
|
if (role.startsWith("/"))
|
|
return line;
|
|
const isInteractive = INTERACTIVE_ROLES.has(roleLower);
|
|
const isContent = CONTENT_ROLES.has(roleLower);
|
|
const isStructural = STRUCTURAL_ROLES.has(roleLower);
|
|
if (options.interactive && !isInteractive)
|
|
return null;
|
|
if (options.compact && isStructural && !name)
|
|
return null;
|
|
const shouldHaveRef = isInteractive || (isContent && name);
|
|
if (shouldHaveRef) {
|
|
const ref = nextRef();
|
|
const resolvedName = isInteractive ? (name ?? "") : name;
|
|
const nth = tracker.getNextIndex(roleLower, resolvedName);
|
|
tracker.trackRef(roleLower, resolvedName, ref);
|
|
refs[ref] = {
|
|
selector: buildSelector(roleLower, resolvedName),
|
|
role: roleLower,
|
|
name: resolvedName,
|
|
nth
|
|
};
|
|
let enhanced = `${prefix}${role}`;
|
|
if (name)
|
|
enhanced += ` "${name}"`;
|
|
enhanced += ` [ref=${ref}]`;
|
|
if (nth > 0)
|
|
enhanced += ` [nth=${nth}]`;
|
|
if (suffix)
|
|
enhanced += suffix;
|
|
return enhanced;
|
|
}
|
|
return line;
|
|
}
|
|
function compactTree(tree) {
|
|
const lines = tree.split("\n");
|
|
const result = [];
|
|
for (let i = 0; i < lines.length; i++) {
|
|
const line = lines[i];
|
|
if (line.includes("[ref=")) {
|
|
result.push(line);
|
|
continue;
|
|
}
|
|
if (line.includes(":") && !line.endsWith(":")) {
|
|
result.push(line);
|
|
continue;
|
|
}
|
|
const currentIndent = getIndentLevel(line);
|
|
let hasRelevantChildren = false;
|
|
for (let j = i + 1; j < lines.length; j++) {
|
|
const childIndent = getIndentLevel(lines[j]);
|
|
if (childIndent <= currentIndent)
|
|
break;
|
|
if (lines[j].includes("[ref=")) {
|
|
hasRelevantChildren = true;
|
|
break;
|
|
}
|
|
}
|
|
if (hasRelevantChildren) {
|
|
result.push(line);
|
|
}
|
|
}
|
|
return result.join("\n");
|
|
}
|
|
export function parseRef(arg) {
|
|
if (arg.startsWith("@"))
|
|
return arg.slice(1);
|
|
if (arg.startsWith("ref="))
|
|
return arg.slice(4);
|
|
if (/^e\\d+$/.test(arg))
|
|
return arg;
|
|
return null;
|
|
}
|