feat: embed daemon JS in binary, eliminate npm install of camoufox-browser
The daemon JS files (daemon.js, server.js, browser.js, protocol.js, snapshot.js, ipc.js) are now EMBEDDED in the compiled Rust binary via include_dir! and extracted to ~/.gyandoot/daemon/ on first run. This eliminates the need to npm install -g camoufox-browser@0.1.0, which bundles an incompatible camoufox-js@0.9.3 that caused: - 'Cannot open database' errors - Re-downloading 298MB browser on every navigation - Circular install/remove/install loops Now the daemon imports camoufox-js from the global npm install (v0.11.0) which is installed by install.sh. Clean, no bundled deps, no conflicts. Also removed: is_npm_package_installed(), install_npm_package(), find_daemon_js() — all replaced by gyandoot_browser::daemon::extract_daemon()
This commit is contained in:
385
crates/gyandoot-browser/daemon/snapshot.js
Normal file
385
crates/gyandoot-browser/daemon/snapshot.js
Normal file
@@ -0,0 +1,385 @@
|
||||
/**
|
||||
* Enhanced snapshot with element refs for deterministic element selection.
|
||||
*
|
||||
* This is largely compatible with agent-browser's snapshot/ref workflow:
|
||||
* - `snapshot -i` returns a tree with `[ref=e1]` markers.
|
||||
* - Subsequent actions can target `@e1` deterministically.
|
||||
*/
|
||||
let refCounter = 0;
|
||||
export function resetRefs() {
|
||||
refCounter = 0;
|
||||
}
|
||||
function nextRef() {
|
||||
return `e${++refCounter}`;
|
||||
}
|
||||
const INTERACTIVE_ROLES = new Set([
|
||||
"button",
|
||||
"link",
|
||||
"textbox",
|
||||
"checkbox",
|
||||
"radio",
|
||||
"combobox",
|
||||
"listbox",
|
||||
"menuitem",
|
||||
"menuitemcheckbox",
|
||||
"menuitemradio",
|
||||
"option",
|
||||
"searchbox",
|
||||
"slider",
|
||||
"spinbutton",
|
||||
"switch",
|
||||
"tab",
|
||||
"treeitem"
|
||||
]);
|
||||
const CONTENT_ROLES = new Set([
|
||||
"heading",
|
||||
"cell",
|
||||
"gridcell",
|
||||
"columnheader",
|
||||
"rowheader",
|
||||
"listitem",
|
||||
"article",
|
||||
"region",
|
||||
"main",
|
||||
"navigation"
|
||||
]);
|
||||
const STRUCTURAL_ROLES = new Set([
|
||||
"generic",
|
||||
"group",
|
||||
"list",
|
||||
"table",
|
||||
"row",
|
||||
"rowgroup",
|
||||
"grid",
|
||||
"treegrid",
|
||||
"menu",
|
||||
"menubar",
|
||||
"toolbar",
|
||||
"tablist",
|
||||
"tree",
|
||||
"directory",
|
||||
"document",
|
||||
"application",
|
||||
"presentation",
|
||||
"none"
|
||||
]);
|
||||
function buildSelector(role, name) {
|
||||
const escapedName = JSON.stringify(name);
|
||||
return `getByRole('${role}', { name: ${escapedName}, exact: true })`;
|
||||
}
|
||||
async function findCursorInteractiveElements(page, selector) {
|
||||
const rootSelector = selector || "body";
|
||||
const scriptBody = `(rootSel) => {
|
||||
const results = [];
|
||||
|
||||
const interactiveRoles = new Set([
|
||||
'button', 'link', 'textbox', 'checkbox', 'radio', 'combobox', 'listbox',
|
||||
'menuitem', 'menuitemcheckbox', 'menuitemradio', 'option', 'searchbox',
|
||||
'slider', 'spinbutton', 'switch', 'tab', 'treeitem'
|
||||
]);
|
||||
|
||||
const interactiveTags = new Set([
|
||||
'a', 'button', 'input', 'select', 'textarea', 'details', 'summary'
|
||||
]);
|
||||
|
||||
const root = document.querySelector(rootSel) || document.body;
|
||||
const allElements = root.querySelectorAll('*');
|
||||
|
||||
const buildSelector = (el) => {
|
||||
const testId = el.getAttribute('data-testid');
|
||||
if (testId) return '[data-testid=\"' + testId + '\"]';
|
||||
if (el.id) return '#' + CSS.escape(el.id);
|
||||
|
||||
const path = [];
|
||||
let current = el;
|
||||
while (current && current !== document.body) {
|
||||
let sel = current.tagName.toLowerCase();
|
||||
const classes = Array.from(current.classList).filter(c => c.trim());
|
||||
if (classes.length > 0) sel += '.' + CSS.escape(classes[0]);
|
||||
|
||||
const parent = current.parentElement;
|
||||
if (parent) {
|
||||
const siblings = Array.from(parent.children);
|
||||
const matching = siblings.filter(s => {
|
||||
if (s.tagName !== current.tagName) return false;
|
||||
if (classes.length > 0 && !s.classList.contains(classes[0])) return false;
|
||||
return true;
|
||||
});
|
||||
if (matching.length > 1) {
|
||||
const idx = matching.indexOf(current) + 1;
|
||||
sel += ':nth-of-type(' + idx + ')';
|
||||
}
|
||||
}
|
||||
path.unshift(sel);
|
||||
current = current.parentElement;
|
||||
if (path.length >= 1) {
|
||||
try {
|
||||
const candidate = path.join(' > ');
|
||||
if (document.querySelectorAll(candidate).length === 1) break;
|
||||
} catch (e) {}
|
||||
}
|
||||
if (path.length >= 10) break;
|
||||
}
|
||||
return path.join(' > ');
|
||||
};
|
||||
|
||||
for (const el of allElements) {
|
||||
const tagName = el.tagName.toLowerCase();
|
||||
if (interactiveTags.has(tagName)) continue;
|
||||
|
||||
const role = el.getAttribute('role');
|
||||
if (role && interactiveRoles.has(role.toLowerCase())) continue;
|
||||
|
||||
const computedStyle = getComputedStyle(el);
|
||||
const hasCursorPointer = computedStyle.cursor === 'pointer';
|
||||
const hasOnClick = el.hasAttribute('onclick') || el.onclick !== null;
|
||||
const tabIndex = el.getAttribute('tabindex');
|
||||
const hasTabIndex = tabIndex !== null && tabIndex !== '-1';
|
||||
|
||||
if (!hasCursorPointer && !hasOnClick && !hasTabIndex) continue;
|
||||
|
||||
if (hasCursorPointer && !hasOnClick && !hasTabIndex) {
|
||||
const parent = el.parentElement;
|
||||
if (parent && getComputedStyle(parent).cursor === 'pointer') continue;
|
||||
}
|
||||
|
||||
const text = (el.textContent || '').trim().slice(0, 100);
|
||||
if (!text) continue;
|
||||
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width === 0 || rect.height === 0) continue;
|
||||
|
||||
results.push({
|
||||
selector: buildSelector(el),
|
||||
text,
|
||||
tagName,
|
||||
hasOnClick,
|
||||
hasCursorPointer,
|
||||
hasTabIndex
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}`;
|
||||
// eslint-disable-next-line @typescript-eslint/no-implied-eval
|
||||
const fn = new Function("return " + scriptBody)();
|
||||
return page.evaluate(fn, rootSelector);
|
||||
}
|
||||
export async function getEnhancedSnapshot(page, options = {}) {
|
||||
resetRefs();
|
||||
const refs = {};
|
||||
const locator = options.selector ? page.locator(options.selector) : page.locator(":root");
|
||||
const ariaTree = await locator.ariaSnapshot();
|
||||
if (!ariaTree) {
|
||||
return { tree: "(empty)", refs: {} };
|
||||
}
|
||||
const enhancedTree = processAriaTree(ariaTree, refs, options);
|
||||
if (options.cursor) {
|
||||
const cursorElements = await findCursorInteractiveElements(page, options.selector);
|
||||
const existingTexts = new Set(Object.values(refs).map((r) => r.name.toLowerCase()));
|
||||
for (const m of enhancedTree.matchAll(/"([^"]+)"/g)) {
|
||||
existingTexts.add(m[1].toLowerCase());
|
||||
}
|
||||
const additionalLines = [];
|
||||
for (const el of cursorElements) {
|
||||
const elTextLower = el.text.toLowerCase();
|
||||
if (existingTexts.has(elTextLower))
|
||||
continue;
|
||||
existingTexts.add(elTextLower);
|
||||
const ref = nextRef();
|
||||
const role = el.hasCursorPointer || el.hasOnClick ? "clickable" : "focusable";
|
||||
refs[ref] = { selector: el.selector, role, name: el.text };
|
||||
const hints = [];
|
||||
if (el.hasCursorPointer)
|
||||
hints.push("cursor:pointer");
|
||||
if (el.hasOnClick)
|
||||
hints.push("onclick");
|
||||
if (el.hasTabIndex)
|
||||
hints.push("tabindex");
|
||||
additionalLines.push(`- ${role} "${el.text}" [ref=${ref}] [${hints.join(", ")}]`);
|
||||
}
|
||||
if (additionalLines.length > 0) {
|
||||
const separator = enhancedTree === "(no interactive elements)"
|
||||
? ""
|
||||
: "\n# Cursor-interactive elements:\n";
|
||||
const base = enhancedTree === "(no interactive elements)" ? "" : enhancedTree;
|
||||
return { tree: base + separator + additionalLines.join("\n"), refs };
|
||||
}
|
||||
}
|
||||
return { tree: enhancedTree, refs };
|
||||
}
|
||||
function createRoleNameTracker() {
|
||||
const counts = new Map();
|
||||
const refsByKey = new Map();
|
||||
return {
|
||||
counts,
|
||||
refsByKey,
|
||||
getKey(role, name) {
|
||||
return `${role}:${name ?? ""}`;
|
||||
},
|
||||
getNextIndex(role, name) {
|
||||
const key = this.getKey(role, name);
|
||||
const current = counts.get(key) ?? 0;
|
||||
counts.set(key, current + 1);
|
||||
return current;
|
||||
},
|
||||
trackRef(role, name, ref) {
|
||||
const key = this.getKey(role, name);
|
||||
const list = refsByKey.get(key) ?? [];
|
||||
list.push(ref);
|
||||
refsByKey.set(key, list);
|
||||
},
|
||||
getDuplicateKeys() {
|
||||
const duplicates = new Set();
|
||||
for (const [key, list] of refsByKey) {
|
||||
if (list.length > 1)
|
||||
duplicates.add(key);
|
||||
}
|
||||
return duplicates;
|
||||
}
|
||||
};
|
||||
}
|
||||
function processAriaTree(ariaTree, refs, options) {
|
||||
const lines = ariaTree.split("\n");
|
||||
const result = [];
|
||||
const tracker = createRoleNameTracker();
|
||||
if (options.interactive) {
|
||||
for (const line of lines) {
|
||||
const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
|
||||
if (!match)
|
||||
continue;
|
||||
const [, , role, name, suffix] = match;
|
||||
const roleLower = role.toLowerCase();
|
||||
if (INTERACTIVE_ROLES.has(roleLower)) {
|
||||
const ref = nextRef();
|
||||
const resolvedName = name ?? "";
|
||||
const nth = tracker.getNextIndex(roleLower, resolvedName);
|
||||
tracker.trackRef(roleLower, resolvedName, ref);
|
||||
refs[ref] = {
|
||||
selector: buildSelector(roleLower, resolvedName),
|
||||
role: roleLower,
|
||||
name: resolvedName,
|
||||
nth
|
||||
};
|
||||
let enhanced = `- ${role}`;
|
||||
if (name)
|
||||
enhanced += ` "${name}"`;
|
||||
enhanced += ` [ref=${ref}]`;
|
||||
if (nth > 0)
|
||||
enhanced += ` [nth=${nth}]`;
|
||||
if (suffix && suffix.includes("["))
|
||||
enhanced += suffix;
|
||||
result.push(enhanced);
|
||||
}
|
||||
}
|
||||
removeNthFromNonDuplicates(refs, tracker);
|
||||
return result.join("\n") || "(no interactive elements)";
|
||||
}
|
||||
for (const line of lines) {
|
||||
const processed = processLine(line, refs, options, tracker);
|
||||
if (processed !== null)
|
||||
result.push(processed);
|
||||
}
|
||||
removeNthFromNonDuplicates(refs, tracker);
|
||||
if (options.compact) {
|
||||
return compactTree(result.join("\n"));
|
||||
}
|
||||
return result.join("\n");
|
||||
}
|
||||
function removeNthFromNonDuplicates(refs, tracker) {
|
||||
const duplicateKeys = tracker.getDuplicateKeys();
|
||||
for (const [ref, data] of Object.entries(refs)) {
|
||||
const key = tracker.getKey(data.role, data.name);
|
||||
if (!duplicateKeys.has(key)) {
|
||||
delete refs[ref].nth;
|
||||
}
|
||||
}
|
||||
}
|
||||
function getIndentLevel(line) {
|
||||
const match = line.match(/^(\s*)/);
|
||||
return match ? Math.floor(match[1].length / 2) : 0;
|
||||
}
|
||||
function processLine(line, refs, options, tracker) {
|
||||
const depth = getIndentLevel(line);
|
||||
if (options.maxDepth !== undefined && depth > options.maxDepth) {
|
||||
return null;
|
||||
}
|
||||
const match = line.match(/^(\s*-\s*)(\w+)(?:\s+"([^"]*)")?(.*)$/);
|
||||
if (!match) {
|
||||
if (options.interactive)
|
||||
return null;
|
||||
return line;
|
||||
}
|
||||
const [, prefix, role, name, suffix] = match;
|
||||
const roleLower = role.toLowerCase();
|
||||
if (role.startsWith("/"))
|
||||
return line;
|
||||
const isInteractive = INTERACTIVE_ROLES.has(roleLower);
|
||||
const isContent = CONTENT_ROLES.has(roleLower);
|
||||
const isStructural = STRUCTURAL_ROLES.has(roleLower);
|
||||
if (options.interactive && !isInteractive)
|
||||
return null;
|
||||
if (options.compact && isStructural && !name)
|
||||
return null;
|
||||
const shouldHaveRef = isInteractive || (isContent && name);
|
||||
if (shouldHaveRef) {
|
||||
const ref = nextRef();
|
||||
const resolvedName = isInteractive ? (name ?? "") : name;
|
||||
const nth = tracker.getNextIndex(roleLower, resolvedName);
|
||||
tracker.trackRef(roleLower, resolvedName, ref);
|
||||
refs[ref] = {
|
||||
selector: buildSelector(roleLower, resolvedName),
|
||||
role: roleLower,
|
||||
name: resolvedName,
|
||||
nth
|
||||
};
|
||||
let enhanced = `${prefix}${role}`;
|
||||
if (name)
|
||||
enhanced += ` "${name}"`;
|
||||
enhanced += ` [ref=${ref}]`;
|
||||
if (nth > 0)
|
||||
enhanced += ` [nth=${nth}]`;
|
||||
if (suffix)
|
||||
enhanced += suffix;
|
||||
return enhanced;
|
||||
}
|
||||
return line;
|
||||
}
|
||||
function compactTree(tree) {
|
||||
const lines = tree.split("\n");
|
||||
const result = [];
|
||||
for (let i = 0; i < lines.length; i++) {
|
||||
const line = lines[i];
|
||||
if (line.includes("[ref=")) {
|
||||
result.push(line);
|
||||
continue;
|
||||
}
|
||||
if (line.includes(":") && !line.endsWith(":")) {
|
||||
result.push(line);
|
||||
continue;
|
||||
}
|
||||
const currentIndent = getIndentLevel(line);
|
||||
let hasRelevantChildren = false;
|
||||
for (let j = i + 1; j < lines.length; j++) {
|
||||
const childIndent = getIndentLevel(lines[j]);
|
||||
if (childIndent <= currentIndent)
|
||||
break;
|
||||
if (lines[j].includes("[ref=")) {
|
||||
hasRelevantChildren = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (hasRelevantChildren) {
|
||||
result.push(line);
|
||||
}
|
||||
}
|
||||
return result.join("\n");
|
||||
}
|
||||
export function parseRef(arg) {
|
||||
if (arg.startsWith("@"))
|
||||
return arg.slice(1);
|
||||
if (arg.startsWith("ref="))
|
||||
return arg.slice(4);
|
||||
if (/^e\\d+$/.test(arg))
|
||||
return arg;
|
||||
return null;
|
||||
}
|
||||
Reference in New Issue
Block a user