Add camofox-browser integration: gyandoot-browser crate + agent tools
New crate: crates/gyandoot-browser/
- client.rs: Full REST client for camofox-browser API (health, tabs,
navigation, snapshots, click/type/scroll, search macros)
- macros.rs: 14 SearchMacro enum (Google, YouTube, Amazon, Reddit,
Wikipedia, Twitter, Yelp, Spotify, Netflix, LinkedIn, Instagram,
TikTok, Twitch) with parse/display
- snapshot.rs: PageSnapshot, PageLinks, NavigationResult, HealthResponse
- session.rs: BrowserSession, SessionManager with idle cleanup
New module: crates/gyandoot-agent/src/browser_tools.rs
- WebSearchTool: Search the web using anti-detection browser
- WebBrowseTool: Browse URLs, click, type, scroll with element refs
- register_browser_tools(): Wire both tools into ToolExecutor
CLI: Added
o
|
- - - - - - .---. - - - - - -
- - - - - - - - - [ ^ ^ ] - - - - - - - - -
- - - - - - - | ___ | - - - - - - -
`-----'
____ __ __ ___ _ _ ____ ___ ___ _____
/ ___|\ \ / / / _ \ | \ | || _ \ / _ \ / _ \|_ _|
| | _ \ V / / /_\ \| \| || | | | | | | | | | | |
| |_| | | | | _ || |\ || |_| | |_| | |_| | | |
\____/ |_| |_| |_||_| \_||____/ \___/ \___/ |_|
o - - - - - - - knowledge. delivered. - - - - - - - o
Unknown command: browser
Run 'gyandoot help' for usage. subcommand with health/search/browse actions
Installer: Updated install.sh with camofox-browser cloning, Node.js
detection, npm dependency install, and systemd service setup
Camofox-browser running at http://localhost:9377 (Docker, verified working)
Workspace: 12 crates + CLI binary — cargo check passes with 0 errors.
This commit is contained in:
@@ -11,6 +11,7 @@ gyandoot-providers = { path = "../gyandoot-providers" }
|
||||
gyandoot-memory = { path = "../gyandoot-memory" }
|
||||
gyandoot-security = { path = "../gyandoot-security" }
|
||||
gyandoot-skills = { path = "../gyandoot-skills" }
|
||||
gyandoot-browser = { path = "../gyandoot-browser" }
|
||||
tokio.workspace = true
|
||||
serde.workspace = true
|
||||
serde_json.workspace = true
|
||||
|
||||
284
crates/gyandoot-agent/src/browser_tools.rs
Normal file
284
crates/gyandoot-agent/src/browser_tools.rs
Normal file
@@ -0,0 +1,284 @@
|
||||
//! Browser tools for the agent — enables web browsing with anti-detection.
|
||||
//!
|
||||
//! These tools are registered with the `ToolExecutor` and provide
|
||||
//! LLM-accessible functions for searching the web, browsing pages,
|
||||
//! clicking elements, and extracting content.
|
||||
|
||||
use serde_json;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use gyandoot_browser::{BrowserClient, SearchMacro};
|
||||
use gyandoot_core::ToolDefinition;
|
||||
|
||||
use crate::tools::Tool;
|
||||
|
||||
/// Tool for searching the web using camofox-browser search macros.
|
||||
pub struct WebSearchTool {
|
||||
client: BrowserClient,
|
||||
}
|
||||
|
||||
impl WebSearchTool {
|
||||
pub fn new(client: BrowserClient) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Tool for WebSearchTool {
|
||||
fn name(&self) -> &str {
|
||||
"web_search"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Search the web using Google, YouTube, Amazon, Reddit, Wikipedia, and more. \
|
||||
Uses anti-detection browser to bypass bot protection. \
|
||||
Returns an accessibility snapshot of search results."
|
||||
}
|
||||
|
||||
fn parameters(&self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"query": {
|
||||
"type": "string",
|
||||
"description": "The search query"
|
||||
},
|
||||
"site": {
|
||||
"type": "string",
|
||||
"description": "Site to search. Options: google, youtube, amazon, reddit, wikipedia, twitter, yelp, spotify, linkedin, instagram, tiktok, twitch",
|
||||
"default": "google"
|
||||
}
|
||||
},
|
||||
"required": ["query"]
|
||||
})
|
||||
}
|
||||
|
||||
async fn execute(&self, args: serde_json::Value) -> anyhow::Result<String> {
|
||||
let query = args["query"].as_str().unwrap_or("").to_string();
|
||||
let site = args["site"].as_str().unwrap_or("google").to_lowercase();
|
||||
|
||||
if query.is_empty() {
|
||||
return Err(anyhow::anyhow!("query is required"));
|
||||
}
|
||||
|
||||
let macro_name = match site.as_str() {
|
||||
"google" => SearchMacro::GoogleSearch,
|
||||
"youtube" => SearchMacro::YoutubeSearch,
|
||||
"amazon" => SearchMacro::AmazonSearch,
|
||||
"reddit" => SearchMacro::RedditSearch,
|
||||
"wikipedia" => SearchMacro::WikipediaSearch,
|
||||
"twitter" | "x" => SearchMacro::TwitterSearch,
|
||||
"yelp" => SearchMacro::YelpSearch,
|
||||
"spotify" => SearchMacro::SpotifySearch,
|
||||
"linkedin" => SearchMacro::LinkedinSearch,
|
||||
"instagram" => SearchMacro::InstagramSearch,
|
||||
"tiktok" => SearchMacro::TiktokSearch,
|
||||
"twitch" => SearchMacro::TwitchSearch,
|
||||
_ => SearchMacro::GoogleSearch,
|
||||
};
|
||||
|
||||
info!("web_search: searching {} for '{}'", macro_name.site_name(), query);
|
||||
|
||||
let session_key = format!("web_search_{}", std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs());
|
||||
|
||||
let tab = self.client.create_tab("gyandoot", &session_key, None).await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to create browser tab: {}", e))?;
|
||||
let tab_id = tab.tab_id;
|
||||
|
||||
let result = self.client.search(&tab_id, "gyandoot", ¯o_name, &query, Some(3000)).await
|
||||
.map_err(|e| anyhow::anyhow!("Search failed: {}", e))?;
|
||||
|
||||
let snap = self.client.snapshot(&tab_id, "gyandoot").await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to get snapshot: {}", e))?;
|
||||
|
||||
// Close the tab to free resources
|
||||
let _ = self.client.close_tab(&tab_id, "gyandoot").await;
|
||||
|
||||
let refs = snap.extract_refs();
|
||||
let response = format!(
|
||||
"## {} Search Results\n\n**Query:** {}\n**Page:** {}\n**Title:** {}\n\n**Snapshot content:**\n{}\n\n**Available element refs:** {:?}\n\n*Use `web_browse` to interact with specific results.*",
|
||||
macro_name.site_name(),
|
||||
query,
|
||||
result.url,
|
||||
result.title,
|
||||
snap.snapshot,
|
||||
refs,
|
||||
);
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
|
||||
/// Tool for browsing a specific URL using camofox-browser.
|
||||
pub struct WebBrowseTool {
|
||||
client: BrowserClient,
|
||||
}
|
||||
|
||||
impl WebBrowseTool {
|
||||
pub fn new(client: BrowserClient) -> Self {
|
||||
Self { client }
|
||||
}
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
impl Tool for WebBrowseTool {
|
||||
fn name(&self) -> &str {
|
||||
"web_browse"
|
||||
}
|
||||
|
||||
fn description(&self) -> &str {
|
||||
"Browse a specific URL using an anti-detection browser. \
|
||||
Returns an accessibility snapshot with element refs for interaction. \
|
||||
Supports navigation, clicking, typing, and scrolling."
|
||||
}
|
||||
|
||||
fn parameters(&self) -> serde_json::Value {
|
||||
serde_json::json!({
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"url": {
|
||||
"type": "string",
|
||||
"description": "The URL to browse"
|
||||
},
|
||||
"action": {
|
||||
"type": "string",
|
||||
"description": "Action to perform: 'navigate' (default), 'click', 'type', 'scroll', 'back', 'forward', 'refresh'",
|
||||
"default": "navigate"
|
||||
},
|
||||
"element_ref": {
|
||||
"type": "string",
|
||||
"description": "Element reference (e.g., 'e1', 'e2') — required for click/type actions"
|
||||
},
|
||||
"text": {
|
||||
"type": "string",
|
||||
"description": "Text to type — required for type action"
|
||||
},
|
||||
"press_enter": {
|
||||
"type": "boolean",
|
||||
"description": "Press Enter after typing",
|
||||
"default": false
|
||||
},
|
||||
"direction": {
|
||||
"type": "string",
|
||||
"description": "Scroll direction: 'up' or 'down'",
|
||||
"default": "down"
|
||||
},
|
||||
"amount": {
|
||||
"type": "integer",
|
||||
"description": "Scroll amount in pixels",
|
||||
"default": 500
|
||||
}
|
||||
},
|
||||
"required": ["url"]
|
||||
})
|
||||
}
|
||||
|
||||
async fn execute(&self, args: serde_json::Value) -> anyhow::Result<String> {
|
||||
let url = args["url"].as_str().unwrap_or("").to_string();
|
||||
let action = args["action"].as_str().unwrap_or("navigate").to_lowercase();
|
||||
let element_ref = args["element_ref"].as_str().map(|s| s.to_string());
|
||||
let text = args["text"].as_str().map(|s| s.to_string());
|
||||
let press_enter = args["press_enter"].as_bool().unwrap_or(false);
|
||||
let direction = args["direction"].as_str().unwrap_or("down").to_string();
|
||||
let amount = args["amount"].as_u64().unwrap_or(500) as u32;
|
||||
|
||||
if url.is_empty() {
|
||||
return Err(anyhow::anyhow!("url is required"));
|
||||
}
|
||||
|
||||
let session_key = format!("web_browse_{}", std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap_or_default()
|
||||
.as_secs());
|
||||
|
||||
let tab = self.client.create_tab("gyandoot", &session_key, Some(&url)).await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to create browser tab: {}", e))?;
|
||||
let tab_id = tab.tab_id;
|
||||
|
||||
match action.as_str() {
|
||||
"click" => {
|
||||
if let Some(ref r#ref) = element_ref {
|
||||
self.client.click(&tab_id, "gyandoot", r#ref).await
|
||||
.map_err(|e| anyhow::anyhow!("Click failed: {}", e))?;
|
||||
} else {
|
||||
return Err(anyhow::anyhow!("element_ref is required for click action"));
|
||||
}
|
||||
}
|
||||
"type" => {
|
||||
if let Some(ref r#ref) = element_ref {
|
||||
if let Some(ref t) = text {
|
||||
self.client.type_text(&tab_id, "gyandoot", r#ref, t, press_enter).await
|
||||
.map_err(|e| anyhow::anyhow!("Type failed: {}", e))?;
|
||||
}
|
||||
} else {
|
||||
return Err(anyhow::anyhow!("element_ref and text are required for type action"));
|
||||
}
|
||||
}
|
||||
"scroll" => {
|
||||
self.client.scroll(&tab_id, "gyandoot", &direction, amount).await
|
||||
.map_err(|e| anyhow::anyhow!("Scroll failed: {}", e))?;
|
||||
}
|
||||
"back" => {
|
||||
self.client.go_back(&tab_id, "gyandoot").await
|
||||
.map_err(|e| anyhow::anyhow!("Go back failed: {}", e))?;
|
||||
}
|
||||
"forward" => {
|
||||
self.client.go_forward(&tab_id, "gyandoot").await
|
||||
.map_err(|e| anyhow::anyhow!("Go forward failed: {}", e))?;
|
||||
}
|
||||
"refresh" => {
|
||||
self.client.refresh(&tab_id, "gyandoot").await
|
||||
.map_err(|e| anyhow::anyhow!("Refresh failed: {}", e))?;
|
||||
}
|
||||
_ => {} // already navigated via create_tab
|
||||
}
|
||||
|
||||
let snap = self.client.snapshot(&tab_id, "gyandoot").await
|
||||
.map_err(|e| anyhow::anyhow!("Failed to get snapshot: {}", e))?;
|
||||
|
||||
let _ = self.client.close_tab(&tab_id, "gyandoot").await;
|
||||
|
||||
let refs = snap.extract_refs();
|
||||
let response = format!(
|
||||
"## Page Snapshot\n\n**URL:** {}\n\n**Snapshot:**\n{}\n\n**Element refs:** {:?}\n\n*Use `web_browse` with action=\"click\" and element_ref to interact with elements.*",
|
||||
url,
|
||||
snap.snapshot,
|
||||
refs,
|
||||
);
|
||||
|
||||
Ok(response)
|
||||
}
|
||||
}
|
||||
|
||||
/// Register all browser tools with the given executor.
|
||||
pub fn register_browser_tools(executor: &mut crate::tools::ToolExecutor, client: BrowserClient) {
|
||||
executor.register(Box::new(WebSearchTool::new(client.clone())));
|
||||
executor.register(Box::new(WebBrowseTool::new(client)));
|
||||
info!("Registered browser tools: web_search, web_browse");
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_web_search_tool_definition() {
|
||||
let client = BrowserClient::new();
|
||||
let tool = WebSearchTool::new(client);
|
||||
assert_eq!(tool.name(), "web_search");
|
||||
let params = tool.parameters();
|
||||
assert!(params["required"].as_array().unwrap().contains(&"query".to_string().into()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_web_browse_tool_definition() {
|
||||
let client = BrowserClient::new();
|
||||
let tool = WebBrowseTool::new(client);
|
||||
assert_eq!(tool.name(), "web_browse");
|
||||
let params = tool.parameters();
|
||||
assert!(params["required"].as_array().unwrap().contains(&"url".to_string().into()));
|
||||
}
|
||||
}
|
||||
@@ -5,6 +5,7 @@
|
||||
//! needed, and interacting with memory.
|
||||
|
||||
pub mod agent_loop;
|
||||
pub mod browser_tools;
|
||||
pub mod tools;
|
||||
pub mod context;
|
||||
|
||||
|
||||
Reference in New Issue
Block a user