cameron.stream
a digital person for bluesky
1"""Webpage fetch tool using Jina AI reader."""
2from typing import Optional
3from pydantic import BaseModel, Field
4
5
6class WebpageArgs(BaseModel):
7 url: str = Field(
8 ...,
9 description="The URL of the webpage to fetch and convert to markdown/text format"
10 )
11
12
13def fetch_webpage(url: str) -> str:
14 """
15 Fetch a webpage and convert it to markdown/text format using Jina AI reader.
16
17 Args:
18 url: The URL of the webpage to fetch and convert
19
20 Returns:
21 String containing the webpage content in markdown/text format
22 """
23 import requests
24 import logging
25
26 logger = logging.getLogger(__name__)
27
28 try:
29 # Construct the Jina AI reader URL
30 jina_url = f"https://r.jina.ai/{url}"
31
32 # Make the request to Jina AI
33 response = requests.get(jina_url, timeout=30)
34 response.raise_for_status()
35
36 logger.info(f"Successfully fetched webpage: {url}")
37 return response.text
38
39 except requests.exceptions.RequestException as e:
40 logger.error(f"Error fetching webpage {url}: {e}")
41 raise Exception(f"Error fetching webpage: {str(e)}")
42 except Exception as e:
43 logger.error(f"Unexpected error fetching webpage {url}: {e}")
44 raise Exception(f"Unexpected error: {str(e)}")