Add read_url tool for web research

agents/context-pruner.ts

@@ -307,6 +307,10 @@ const definition: AgentDefinition = {
             ? `web search for "${query}"`
             : 'web search'
         }
+        case 'read_url': {
+          const url = input.url as string | undefined
+          return url ? `read URL: ${url}` : 'read a URL'
+        }
         case 'gravity_index': {
           const query = input.query as string | undefined
           const action = input.action as string | undefined

agents/researcher/researcher-web.ts

@@ -16,15 +16,17 @@ const definition: SecretAgentDefinition = {
   },
   outputMode: 'last_message',
   includeMessageHistory: false,
-  toolNames: ['web_search', 'run_terminal_command'],
+  toolNames: ['web_search', 'read_url'],
   spawnableAgents: [],
 
-  systemPrompt: `You are an expert researcher who can search the web to find relevant information. Your goal is to answer the user's question from current search results and any useful source pages. Use web_search to get Serper JSON search results. Use run_terminal_command with tools like curl to fetch web pages that would help answer the user's question.`,
+  systemPrompt: `You are an expert researcher who can search the web to find relevant information. Your goal is to answer the user's question from current search results and useful source pages. Use web_search to get Serper JSON search results. Use read_url to fetch and extract readable text from pages that would help answer the user's question.`,
   instructionsPrompt: `Provide comprehensive research on the user's prompt.
 
 Use web_search to find current information. The tool returns JSON search results, so inspect the titles, links, snippets, answer boxes, and related results before deciding what to fetch next.
 
-Use run_terminal_command to fetch any web page that would help answer the user's question. Prefer targeted, relevant pages from the search results. Avoid fetching pages that are unlikely to add useful evidence.
+Use read_url to fetch any web page that would help answer the user's question. Prefer targeted, relevant pages from the search results, especially official or primary sources. Avoid fetching pages that are unlikely to add useful evidence.
+
+If read_url cannot handle a source, choose a different result or explain the limitation.
 
 Then, write up a concise answer that includes key findings for the user's prompt and cites source URLs when useful.
 `.trim(),

agents/types/agent-definition.ts

@@ -345,7 +345,7 @@ export type TerminalTools = 'run_terminal_command' | 'code_search'
 /**
  * Web and browser tools
  */
-export type WebTools = 'web_search' | 'read_docs'
+export type WebTools = 'web_search' | 'read_docs' | 'read_url'
 
 /**
  * Agent management tools

agents/types/tools.ts

@@ -17,6 +17,7 @@ export type ToolName =
   | 'read_docs'
   | 'read_files'
   | 'read_subtree'
+  | 'read_url'
   | 'render_ui'
   | 'run_file_change_hooks'
   | 'run_terminal_command'
@@ -51,6 +52,7 @@ export interface ToolParamsMap {
   read_docs: ReadDocsParams
   read_files: ReadFilesParams
   read_subtree: ReadSubtreeParams
+  read_url: ReadUrlParams
   render_ui: RenderUiParams
   run_file_change_hooks: RunFileChangeHooksParams
   run_terminal_command: RunTerminalCommandParams
@@ -276,6 +278,16 @@ export interface ReadSubtreeParams {
   maxTokens?: number
 }
 
+/**
+ * Fetch a URL and extract readable text from the page.
+ */
+export interface ReadUrlParams {
+  /** The full http:// or https:// URL to fetch and extract readable text from. */
+  url: string
+  /** Maximum number of extracted text characters to return. Defaults to 20000. */
+  max_chars?: number
+}
+
 /**
  * Render a small interactive UI widget in the Codebuff CLI. Currently supports a button that opens a link.
  */

common/src/templates/initial-agents-dir/README.md

@@ -132,6 +132,7 @@ export default {
 ### Web & Research
 
 - **`web_search`**: Search the internet for information
+- **`read_url`**: Fetch a URL and extract readable page text
 - **`read_docs`**: Read technical documentation
 - **`browser_logs`**: Navigate and inspect web pages
 

common/src/templates/initial-agents-dir/types/agent-definition.ts

@@ -345,7 +345,7 @@ export type TerminalTools = 'run_terminal_command' | 'code_search'
 /**
  * Web and browser tools
  */
-export type WebTools = 'web_search' | 'read_docs'
+export type WebTools = 'web_search' | 'read_docs' | 'read_url'
 
 /**
  * Agent management tools

common/src/templates/initial-agents-dir/types/tools.ts

@@ -17,6 +17,7 @@ export type ToolName =
   | 'read_docs'
   | 'read_files'
   | 'read_subtree'
+  | 'read_url'
   | 'render_ui'
   | 'run_file_change_hooks'
   | 'run_terminal_command'
@@ -51,6 +52,7 @@ export interface ToolParamsMap {
   read_docs: ReadDocsParams
   read_files: ReadFilesParams
   read_subtree: ReadSubtreeParams
+  read_url: ReadUrlParams
   render_ui: RenderUiParams
   run_file_change_hooks: RunFileChangeHooksParams
   run_terminal_command: RunTerminalCommandParams
@@ -276,6 +278,16 @@ export interface ReadSubtreeParams {
   maxTokens?: number
 }
 
+/**
+ * Fetch a URL and extract readable text from the page.
+ */
+export interface ReadUrlParams {
+  /** The full http:// or https:// URL to fetch and extract readable text from. */
+  url: string
+  /** Maximum number of extracted text characters to return. Defaults to 20000. */
+  max_chars?: number
+}
+
 /**
  * Render a small interactive UI widget in the Codebuff CLI. Currently supports a button that opens a link.
  */

common/src/tools/constants.ts

@@ -39,6 +39,7 @@ export const toolNames = [
   'read_docs',
   'read_files',
   'read_subtree',
+  'read_url',
   'render_ui',
   'run_file_change_hooks',
   'run_terminal_command',
@@ -73,6 +74,7 @@ export const publishedTools = [
   'read_docs',
   'read_files',
   'read_subtree',
+  'read_url',
   'render_ui',
   'run_file_change_hooks',
   'run_terminal_command',

common/src/tools/list.ts

@@ -19,6 +19,7 @@ import { proposeWriteFileParams } from './params/tool/propose-write-file'
 import { readDocsParams } from './params/tool/read-docs'
 import { readFilesParams } from './params/tool/read-files'
 import { readSubtreeParams } from './params/tool/read-subtree'
+import { readUrlParams } from './params/tool/read-url'
 import { renderUIParams } from './params/tool/render-ui'
 import { runFileChangeHooksParams } from './params/tool/run-file-change-hooks'
 import { runTerminalCommandParams } from './params/tool/run-terminal-command'
@@ -59,6 +60,7 @@ export const toolParams = {
   read_docs: readDocsParams,
   read_files: readFilesParams,
   read_subtree: readSubtreeParams,
+  read_url: readUrlParams,
   render_ui: renderUIParams,
   run_file_change_hooks: runFileChangeHooksParams,
   run_terminal_command: runTerminalCommandParams,
@@ -131,6 +133,10 @@ export const clientToolCallSchema = z.discriminatedUnion('toolName', [
     toolName: z.literal('run_file_change_hooks'),
     input: toolParams.run_file_change_hooks.inputSchema,
   }),
+  z.object({
+    toolName: z.literal('read_url'),
+    input: toolParams.read_url.inputSchema,
+  }),
   z.object({
     toolName: z.literal('run_terminal_command'),
     input: toolParams.run_terminal_command.inputSchema.and(

common/src/tools/params/tool/read-url.ts

@@ -0,0 +1,81 @@
+import z from 'zod/v4'
+
+import { $getNativeToolCallExampleString, jsonToolResultSchema } from '../utils'
+
+import type { $ToolParams } from '../../constants'
+
+const toolName = 'read_url'
+const endsAgentStep = true
+const inputSchema = z
+  .object({
+    url: z
+      .url()
+      .refine((value) => {
+        try {
+          const parsedUrl = new URL(value)
+          return (
+            parsedUrl.protocol === 'http:' || parsedUrl.protocol === 'https:'
+          )
+        } catch {
+          return false
+        }
+      }, 'URL must use http:// or https://')
+      .describe(
+        'The full http:// or https:// URL to fetch and extract readable text from.',
+      ),
+    max_chars: z
+      .number()
+      .int()
+      .min(1_000)
+      .max(50_000)
+      .default(20_000)
+      .optional()
+      .describe(
+        'Maximum number of extracted text characters to return. Defaults to 20000.',
+      ),
+  })
+  .describe('Fetch a URL and extract readable text from the page.')
+
+const description = `
+Purpose: Fetch a URL returned by web_search and extract the readable page text so you can answer with source-backed evidence.
+
+Use this after web_search when snippets are not enough. Prefer authoritative, relevant pages from the search results. The tool follows redirects, extracts titles and metadata, strips scripts/styles/navigation boilerplate from HTML, and returns normalized readable text.
+
+Do not use run_terminal_command with curl just to inspect web pages; use read_url instead. If read_url reports unsupported content or extraction failure, then choose a different search result or explain the limitation.
+
+Example:
+${$getNativeToolCallExampleString({
+  toolName,
+  inputSchema,
+  input: {
+    url: 'https://react.dev/reference/react/useActionState',
+    max_chars: 12000,
+  },
+  endsAgentStep,
+})}
+`.trim()
+
+export const readUrlParams = {
+  toolName,
+  endsAgentStep,
+  description,
+  inputSchema,
+  outputSchema: jsonToolResultSchema(
+    z.union([
+      z.object({
+        url: z.string(),
+        finalUrl: z.string(),
+        status: z.number(),
+        contentType: z.string().optional(),
+        title: z.string().optional(),
+        description: z.string().optional(),
+        text: z.string(),
+        truncated: z.boolean(),
+      }),
+      z.object({
+        url: z.string().optional(),
+        errorMessage: z.string(),
+      }),
+    ]),
+  ),
+} satisfies $ToolParams

packages/agent-runtime/src/tools/handlers/list.ts

@@ -16,6 +16,7 @@ import { handleProposeWriteFile } from './tool/propose-write-file'
 import { handleReadDocs } from './tool/read-docs'
 import { handleReadFiles } from './tool/read-files'
 import { handleReadSubtree } from './tool/read-subtree'
+import { handleReadUrl } from './tool/read-url'
 import { handleRenderUI } from './tool/render-ui'
 import { handleRunFileChangeHooks } from './tool/run-file-change-hooks'
 import { handleRunTerminalCommand } from './tool/run-terminal-command'
@@ -64,6 +65,7 @@ export const codebuffToolHandlers = {
   read_docs: handleReadDocs,
   read_files: handleReadFiles,
   read_subtree: handleReadSubtree,
+  read_url: handleReadUrl,
   render_ui: handleRenderUI,
   run_file_change_hooks: handleRunFileChangeHooks,
   run_terminal_command: handleRunTerminalCommand,

packages/agent-runtime/src/tools/handlers/tool/read-url.ts

@@ -0,0 +1,21 @@
+import type { CodebuffToolHandlerFunction } from '../handler-function-type'
+import type {
+  ClientToolCall,
+  CodebuffToolCall,
+  CodebuffToolOutput,
+} from '@codebuff/common/tools/list'
+
+export const handleReadUrl = (async (params: {
+  previousToolCallFinished: Promise<void>
+  toolCall: CodebuffToolCall<'read_url'>
+  requestClientToolCall: (
+    toolCall: ClientToolCall<'read_url'>,
+  ) => Promise<CodebuffToolOutput<'read_url'>>
+}): Promise<{
+  output: CodebuffToolOutput<'read_url'>
+}> => {
+  const { previousToolCallFinished, toolCall, requestClientToolCall } = params
+
+  await previousToolCallFinished
+  return { output: await requestClientToolCall(toolCall) }
+}) satisfies CodebuffToolHandlerFunction<'read_url'>

packages/agent-runtime/src/tools/tool-executor.ts

@@ -62,6 +62,7 @@ const bareStringFieldRepairAllowlist: Partial<
   lookup_agent_info: ['agentId'],
   read_files: ['paths'],
   read_subtree: ['paths'],
+  read_url: ['url'],
   skill: ['name'],
   web_search: ['query'],
 }