| | import os |
| | from scrapingbee import ScrapingBeeClient |
| | from logger import setup_logger |
| | import json |
| |
|
| | logger = setup_logger("scraper") |
| |
|
| | |
| | client = ScrapingBeeClient(api_key=os.getenv('SCRAPINGBEE_API_KEY', '')) |
| |
|
| | def scrape_url(url: str) -> str: |
| | """ |
| | Scrape content from URL using ScrapingBee with AI extraction |
| | |
| | Args: |
| | url: The URL to scrape |
| | |
| | Returns: |
| | str: Extracted text content or error message |
| | """ |
| | try: |
| | logger.info(f"Scraping URL: {url}") |
| | response = client.get( |
| | url, |
| | params={ |
| | 'stealth_proxy': True, |
| | 'country_code': 'us', |
| | 'ai_query': 'Extract the main text content from this page' |
| | } |
| | ) |
| | |
| | if response.status_code == 200: |
| | logger.info(f"Successfully scraped URL: {url}") |
| | return response.text if response.text else "No content could be extracted from the URL" |
| | else: |
| | logger.error(f"Failed to scrape URL: {url}, Status: {response.status_code}") |
| | return f"Failed to download the URL. Status code: {response.status_code}" |
| | |
| | except Exception as e: |
| | logger.error(f"Error scraping URL: {url}", exc_info=True) |
| | return f"Error scraping the URL: {str(e)}" |
