LangChain AI Agents for Browser Automation

// Hardcoded steps - brittle
await page.goto('https://example.com')
await page.click('#search-input')
await page.type('search-input', 'query')
await page.click('#search-button')
await page.waitForNavigation()

// Natural language goal - flexible
await agent.executeTask('Go to example.com, search for "query", and get results')

User Prompt
    ↓
LangGraph ReAct Agent
    ↓
   Reasoning ←→ Browser Tools
    ↓
Puppeteer/Playwright Page
    ↓
Website (via CDP)

import type { BaseChatModel } from '@langchain/core/language_models/chat_models'
import { createReactAgent } from '@langchain/langgraph/prebuilt'
import type { Page } from 'puppeteer-core'

export class BrowserAgent {
  private page: Page
  private llm: BaseChatModel
  private tools: LangChainTool[]
  private agent: AgentExecutor

  constructor(config: { page: Page; browserUrl: string; model: BaseChatModel }) {
    this.page = config.page
    this.llm = config.model
    this.tools = createBrowserTools(this.page)
    this.agent = createReactAgent({ llm: this.llm, tools: this.tools })
  }

  async executeTask(task: string): Promise<ExecuteTaskResult> {
    const result = await this.agent.invoke({
      messages: [{ role: 'user', content: task }],
    })
    return result
  }
}

npm install @langchain/core @langchain/langgraph
npm install puppeteer-core  # or: npm install playwright
npm install zod

npm install @langchain/openai      # OpenAI, Groq, Together, etc.
npm install @langchain/anthropic   # Claude
npm install @langchain/google-genai # Gemini

npm install playwright
npx playwright install chromium

import { DynamicStructuredTool } from '@langchain/core/tools'
import { z } from 'zod'
import type { Page } from 'puppeteer-core'

export const createBrowserTools = (page: Page) => {
  return [
    // Navigate Tool
    new DynamicStructuredTool({
      name: 'navigate',
      description: 'Navigate to a URL',
      schema: z.object({
        url: z.string().url().describe('Full URL to navigate to'),
      }),
      func: async ({ url }) => {
        await page.goto(url, { waitUntil: 'networkidle2', timeout: 30000 })
        return `Navigated to ${url}`
      },
    }),

    // Click Tool
    new DynamicStructuredTool({
      name: 'click_element',
      description: 'Click element by CSS selector',
      schema: z.object({
        selector: z.string().describe('CSS selector to click'),
        waitForNavigation: z.boolean().optional().describe('Wait after clicking'),
      }),
      func: async ({ selector, waitForNavigation = false }) => {
        await page.waitForSelector(selector, { visible: true, timeout: 10000 })
        if (waitForNavigation) {
          await Promise.all([
            page.waitForNavigation({ waitUntil: 'networkidle2' }),
            page.click(selector),
          ])
        } else {
          await page.click(selector)
        }
        return `Clicked ${selector}`
      },
    }),

    // Type Tool
    new DynamicStructuredTool({
      name: 'type_text',
      description: 'Type text into input field',
      schema: z.object({
        selector: z.string().describe('CSS selector for input'),
        text: z.string().describe('Text to type'),
        clearFirst: z.boolean().optional().describe('Clear before typing'),
      }),
      func: async ({ selector, text, clearFirst = true }) => {
        await page.waitForSelector(selector, { visible: true, timeout: 10000 })
        if (clearFirst) {
          await page.click(selector, { clickCount: 3 })
          await page.keyboard.press('Backspace')
        }
        await page.type(selector, text)
        return `Typed "${text}" into ${selector}`
      },
    }),

    // Get Page Info Tool
    new DynamicStructuredTool({
      name: 'get_page_info',
      description: 'Get current page information',
      schema: z.object({}),
      func: async () => {
        const url = page.url()
        const title = await page.title()
        const bodyText = await page.evaluate(() => {
          return document.body.innerText.substring(0, 500)
        })
        return `URL: ${url}, Title: ${title}, Text: ${bodyText}...`
      },
    }),
  ]
}

// Good
description: 'Navigate to a specific URL. Use this when you need to go to a new webpage.'

// Bad
description: 'Nav'

schema: z.object({
  url: z.string().url(), // Must be valid URL
  timeout: z.number().min(1000).max(60000), // Must be 1-60s
})

func: async ({ selector }) => {
  try {
    await page.click(selector)
    return `Success`
  } catch (error) {
    return `Error: ${error.message}` // Return error to agent
  }
}

import { ChatOpenAI } from '@langchain/openai'
import { ChatGoogleGenerativeAI } from '@langchain/google-genai'

// OpenAI
const llm = new ChatOpenAI({
  modelName: 'gpt-4o-mini',
  temperature: 0,
  openAIApiKey: process.env.OPENAI_API_KEY,
})

// Gemini
const llm = new ChatGoogleGenerativeAI({
  model: 'gemini-2.0-flash',
  temperature: 0,
  apiKey: process.env.GEMINI_API_KEY,
})

// Claude (Anthropic)
import { ChatAnthropic } from '@langchain/anthropic'

const llm = new ChatAnthropic({
  model: 'claude-3-5-sonnet-20241022',
  temperature: 0,
  anthropicApiKey: process.env.ANTHROPIC_API_KEY,
})

// Any OpenAI-compatible API (Groq, Together, local Ollama, etc.)
const llm = new ChatOpenAI({
  modelName: 'llama-3.3-70b-versatile',
  temperature: 0,
  openAIApiKey: process.env.GROQ_API_KEY,
  configuration: {
    baseURL: 'https://api.groq.com/openai/v1',
  },
})

import { createReactAgent } from '@langchain/langgraph/prebuilt'

export class BrowserAgent {
  private page: Page
  private llm: BaseChatModel
  private tools: LangChainTool[]
  private agent: AgentExecutor
  private maxSteps: number

  constructor(config: { page: Page; browserUrl: string; model: BaseChatModel; maxSteps?: number }) {
    this.page = config.page
    this.llm = config.model
    this.maxSteps = config.maxSteps || 50
    this.tools = createBrowserTools(this.page)

    // Create ReAct agent (Reasoning + Acting)
    this.agent = createReactAgent({
      llm: this.llm,
      tools: this.tools,
    })

    console.log(`BrowserAgent initialized with model: ${config.model.modelName}`)
  }

  async executeTask(task: string, options: { verbose?: boolean } = {}): Promise<ExecuteTaskResult> {
    console.log(`Executing task: "${task}"`)

    try {
      const startTime = Date.now()
      const steps: Step[] = []

      // Execute agent
      const result = await this.agent.invoke(
        {
          messages: [{ role: 'user', content: task }],
        },
        {
          recursionLimit: this.maxSteps,
          // Track steps for debugging
          callbacks: [
            {
              handleToolStart: (tool, input, _runId) => {
                const step = steps.length + 1
                if (options.verbose) {
                  console.log(`Step ${step}: ${tool.name}(${JSON.stringify(input)})`)
                }
                steps.push({
                  step,
                  tool: tool.name,
                  input,
                  timestamp: new Date().toISOString(),
                })
              },
              handleToolEnd: (output) => {
                if (steps.length > 0) {
                  steps[steps.length - 1].output = String(output)
                }
                if (options.verbose) {
                  console.log(`  → ${output}`)
                }
              },
            },
          ],
        }
      )

      const duration = Date.now() - startTime

      // Extract token usage
      const lastMessage = result.messages[result.messages.length - 1]
      const tokenUsage = lastMessage?.usage_metadata

      return {
        success: true,
        output: String(result.messages[result.messages.length - 1].content),
        steps,
        duration,
        stepsCount: steps.length,
        tokenUsage,
      }
    } catch (error) {
      return {
        success: false,
        error: error.message,
      }
    }
  }
}

callbacks: [
  {
    handleToolStart: (tool, input) => {
      console.log(`Step ${step}: ${tool.name}`)
    },
    handleToolEnd: (output) => {
      console.log(`  → ${output}`)
    },
  },
]

{
  recursionLimit: this.maxSteps, // Stop after N steps
}

const tokenUsage = result.messages[result.messages.length - 1]?.usage_metadata
// { prompt_tokens: 1234, completion_tokens: 5678, total_tokens: 6912 }

const result = await agent.executeTask('Navigate to https://example.com')
console.log(result.output)
// "Navigated to https://example.com"

const result = await agent.executeTask(
  'Go to https://google.com, search for "Puppeteer", and click the first result'
)

console.log(`Steps: ${result.stepsCount}`)
console.log(`Duration: ${result.duration}ms`)
console.log(`Tokens: ${result.tokenUsage?.total_tokens}`)

const result = await agent.executeTask(
  'Go to https://news.ycombinator.com, extract the top 5 headlines, and return them'
)

console.log(result.output)
// "1. AI Breaks Records...
// 2. New Study Shows...
// 3. OpenAI Releases...
// 4. ..."

const result = await agent.executeTask(
  'Go to https://example.com/signup, fill in the form with name "John" and email "[email protected]", and submit'
)

const tasks = {
  login: (email: string, password: string) =>
    `Go to https://example.com/login, enter email "${email}" and password "${password}", and click login`,

  scrapeListings: (url: string, count: number) =>
    `Go to ${url}, find the first ${count} product cards, extract title, price, and image URL`,

  postStatus: (content: string) =>
    `Navigate to Facebook, click "What's on your mind", type "${content}", and press Ctrl+Enter to post`,
}

// Use template
await agent.executeTask(tasks.login('[email protected]', 'password123'))

async function executeWithRetry(
  agent: BrowserAgent,
  task: string,
  maxRetries = 3
): Promise<ExecuteTaskResult> {
  for (let attempt = 0; attempt < maxRetries; attempt++) {
    const result = await agent.executeTask(task)

    if (result.success) {
      return result
    }

    console.log(`Attempt ${attempt + 1} failed: ${result.error}`)

    // Screenshot for debugging
    if (attempt === maxRetries - 1) {
      await page.screenshot({ path: 'error.png' })
    }

    await new Promise((resolve) => setTimeout(resolve, 2000))
  }

  return { success: false, error: 'Max retries exceeded' }
}

const context = `
You are browsing Facebook Marketplace.
- You are logged in as a seller.
- Your goal is to post a vehicle listing.
- The form has fields: title, price, description, condition, location.
- Use "Post" button to submit when all fields are filled.
`

const result = await agent.executeTask(context + '\n\n' + task)

callbacks: [
  {
    handleToolStart: (tool, input) => {
      // Prevent dangerous actions
      if (tool.name === 'navigate' && input.url.includes('delete')) {
        throw new Error('Navigation to delete URL prevented')
      }

      // Log for audit
      auditLog.push({
        timestamp: new Date(),
        tool: tool.name,
        input,
      })
    },
  },
]

import puppeteer from 'puppeteer-core'
import { ChatOpenAI } from '@langchain/openai'
import { createReactAgent } from '@langchain/langgraph/prebuilt'
import { createBrowserTools } from './browserTools'

async function postFacebookStatus(content: string) {
  // 1. Launch browser
  const browser = await puppeteer.launch({ headless: false })
  const page = await browser.newPage()

  // 2. Create agent
  const llm = new ChatOpenAI({
    modelName: 'gpt-4o-mini',
    temperature: 0,
    openAIApiKey: process.env.OPENAI_API_KEY,
  })

  const agent = new BrowserAgent({
    page,
    browserUrl: 'http://localhost:9222',
    model: llm,
    maxSteps: 30,
  })

  // 3. Define task
  const task = `You are an autonomous browser agent controlling a logged-in Facebook session.

OBJECTIVE: Publish the following content as a new post:
"${content}"

EXECUTION STRATEGY:
1. Navigate to https://www.facebook.com
2. Wait for page to load using get_page_info()
3. Find the post composer (search for "What's on your mind")
4. Click the composer
5. Wait for modal to open (up to 10 seconds)
6. Type the content into the contenteditable div
7. Wait 2-3 seconds
8. Click the "Post" button (use force_click)
9. Wait 3 seconds to verify post was published
10. Declare success and STOP

CORE PRINCIPLES:
- Use find_elements_by_text() for discovery over CSS selectors
- Wait for elements before interacting (3-5 second timeouts)
- If an action fails, retry up to 3 times with 2-second delays
- Stay within facebook.com
- Should take approximately 15-20 steps total
- If you reach 25+ steps, STOP and report issue`

  // 4. Execute
  const result = await agent.executeTask(task, { verbose: true })

  console.log(`Steps: ${result.stepsCount}`)
  console.log(`Duration: ${result.duration}ms`)
  console.log(`Tokens: ${result.tokenUsage?.total_tokens}`)

  // 5. Cleanup
  await browser.close()

  return result
}

// Usage
postFacebookStatus('Hello from automated posting!')

if (result.stepsCount >= maxSteps) {
  console.warn('Agent reached max steps without completing task')
  // Screenshot for debugging
  await page.screenshot({ path: 'max-steps-error.png' })
}

// Check if any step failed
const failedSteps = result.steps.filter(
  (s) => s.output?.toLowerCase().includes('error') || s.output?.toLowerCase().includes('failed')
)

if (failedSteps.length > 0) {
  console.warn('Agent encountered errors:', failedSteps)
}

// Verify agent actually used tools
if (result.stepsCount === 0) {
  console.warn('Agent did not execute any tools')
}

// Verify final state
const finalUrl = page.url()
if (!finalUrl.includes('expected-domain.com')) {
  console.warn('Agent did not reach expected destination')
}

const result = await agent.executeTask(task, { verbose: true })
// Shows each step in real-time

if (!result.success) {
  await page.screenshot({ path: `error-${Date.now()}.png` })
  await page.content().then((html) => fs.writeFileSync(`error-${Date.now()}.html`, html))
}

result.steps.forEach((step, index) => {
  console.log(`Step ${index + 1}:`)
  console.log(`  Tool: ${step.tool}`)
  console.log(`  Input: ${JSON.stringify(step.input)}`)
  console.log(`  Output: ${step.output}`)
  console.log(`  Timestamp: ${step.timestamp}`)
})

// Bad - Too verbose
const task = `Please navigate to the website located at https://example.com. Once you are there, find the search input field and type the text "query". Then click the search button to submit the search.`

// Good - Concise
const task = `Navigate to https://example.com, search for "query"`

// Tool descriptions are sent with every call
// Keep them short and focused
description: 'Navigate to URL' // Good
description: 'Navigate to a specific URL. Use this when you need to go to a new webpage or different section.' // Verbose

// Start with lower limit for testing
const agent = new BrowserAgent({
  page,
  model: llm,
  maxSteps: 15, // Increase after testing
})

// Increase for complex tasks
const agent = new BrowserAgent({
  page,
  model: llm,
  maxSteps: 50,
})

describe('clickTool', () => {
  it('should click element successfully', async () => {
    const mockPage = {
      waitForSelector: jest.fn(),
      click: jest.fn().mockResolvedValue(undefined),
    }
    const tool = createBrowserTools(mockPage)[1] // clickTool

    const result = await tool.func({ selector: '#button' })

    expect(result).toContain('Successfully clicked')
    expect(mockPage.click).toHaveBeenCalledWith('#button')
  })
})

describe('BrowserAgent', () => {
  it('should navigate and extract text', async () => {
    const browser = await puppeteer.launch({ headless: true })
    const page = await browser.newPage()
    const agent = new BrowserAgent({ page, browserUrl: 'ws://...', model: llm })

    const result = await agent.executeTask('Go to https://example.com and extract the main heading')

    expect(result.success).toBe(true)
    expect(result.output).toContain('Example Domain')

    await browser.close()
  })
})

describe('Facebook Posting', () => {
  it('should post status successfully', async () => {
    const result = await postFacebookStatus('Test post')

    expect(result.success).toBe(true)
    expect(result.stepsCount).toBeLessThan(30)
    expect(result.duration).toBeLessThan(60000) // 60 seconds
  })
})

const task = `... Stop once you see confirmation message and DO NOT continue.`

// Instead of: click_element('.obfuscated-class')
// Agent uses: find_elements_by_text('Submit')

const agent = new BrowserAgent({
  model: new ChatOpenAI({ modelName: 'gpt-4o-mini' }),
  maxSteps: 20,
})