| import json |
|
|
| import requests |
| import streamlit as st |
| from crewai import Agent, Task |
| from langchain.tools import tool |
| from unstructured.partition.html import partition_html |
|
|
|
|
| class BrowserTools(): |
|
|
| @tool("Scrape website content") |
| def scrape_and_summarize_website(website): |
| """Useful to scrape and summarize a website content""" |
| url = f"http://api.scrapingant.com/v2/general?url{website}&x-api-key={st.secrets['SCRAPINGANT_API_KEY']}" |
| payload = json.dumps({"url": website}) |
| headers = {'cache-control': 'no-cache', 'content-type': 'application/json'} |
| response = requests.request("GET", url, headers=headers, data=payload) |
| elements = partition_html(text=response.text) |
| content = "\n\n".join([str(el) for el in elements]) |
| content = [content[i:i + 8000] for i in range(0, len(content), 8000)] |
| summaries = [] |
| for chunk in content: |
| agent = Agent( |
| role='Principal Researcher', |
| goal= |
| 'Do amazing researches and summaries based on the content you are working with', |
| backstory= |
| "You're a Principal Researcher at a big company and you need to do a research about a given topic.", |
| allow_delegation=False) |
| task = Task( |
| agent=agent, |
| description= |
| f'Analyze and summarize the content bellow, make sure to include the most relevant information in the summary, return only the summary nothing else.\n\nCONTENT\n----------\n{chunk}' |
| ) |
| summary = task.execute() |
| summaries.append(summary) |
| return "\n\n".join(summaries) |
|
|
