Pre/beta

README.md

@@ -32,26 +32,38 @@ playwright install
 
 **Note**: it is recommended to install the library in a virtual environment to avoid conflicts with other libraries 🐱
 
-By the way if you to use not mandatory modules it is necessary to install by yourself with the following command:
+<details>
+<summary><b>Optional Dependencies</b></summary>
+Additional dependecies can be added while installing the library:
 
-### Installing "Other Language Models"
+- <b>More Language Models</b>: additional language models are installed, such as Fireworks, Groq, Anthropic, Hugging Face, and Nvidia AI Endpoints.
 
-This group allows you to use additional language models like Fireworks, Groq, Anthropic, Hugging Face, and Nvidia AI Endpoints.
+
+This group allows you to use additional language models like Fireworks, Groq, Anthropic, Together AI, Hugging Face, and Nvidia AI Endpoints.
 ```bash
 pip install scrapegraphai[other-language-models]
 
-```
-### Installing "More Semantic Options"
+- <b>Semantic Options</b>: this group includes tools for advanced semantic processing, such as Graphviz.
+
+  ```bash
+  pip install scrapegraphai[more-semantic-options]
+  ```
+
+- <b>Browsers Options</b>: this group includes additional browser management tools/services, such as Browserbase.
+
+  ```bash
+  pip install scrapegraphai[more-browser-options]
+  ```
+
+</details>
+
+
 
-This group includes tools for advanced semantic processing, such as Graphviz.
-```bash
-pip install scrapegraphai[more-semantic-options]
-```
 ### Installing "More Browser Options"
 
-This group includes additional browser management options, such as BrowserBase.
+This group includes an ocr scraper for websites
 ```bash
-pip install scrapegraphai[more-browser-options]
+pip install scrapegraphai[screenshot_scraper]
 ```
 
 ## 💻 Usage
@@ -68,7 +80,7 @@ from scrapegraphai.graphs import SmartScraperGraph
 graph_config = {
     "llm": {
         "api_key": "YOUR_OPENAI_APIKEY",
-        "model": "gpt-4o-mini",
+        "model": "openai/gpt-4o-mini",
     },
     "verbose": True,
     "headless": False,
@@ -128,6 +140,9 @@ Check out also the Docusaurus [here](https://scrapegraph-doc.onrender.com/).
 
 ## 🏆 Sponsors
 <div style="text-align: center;">
+  <a href="https://2ly.link/1zaXG">
+    <img src="https://raw.githubusercontent.com/VinciGit00/Scrapegraph-ai/main/docs/assets/browserbase_logo.png" alt="Browserbase" style="width: 10%;">
+  </a>
   <a href="https://2ly.link/1zNiz">
     <img src="https://raw.githubusercontent.com/VinciGit00/Scrapegraph-ai/main/docs/assets/serp_api_logo.png" alt="SerpAPI" style="width: 10%;">
   </a>

docs/source/introduction/overview.rst

@@ -82,6 +82,11 @@ FAQ
 Sponsors
 ========
 
+.. image:: ../../assets/browserbase_logo.png
+   :width: 10%
+   :alt: Browserbase
+   :target: https://www.browserbase.com/
+
 .. image:: ../../assets/serp_api_logo.png
    :width: 10%
    :alt: Serp API

examples/anthropic/csv_scraper_graph_multi_haiku.py

@@ -26,8 +26,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000},
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

examples/anthropic/csv_scraper_haiku.py

@@ -32,9 +32,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

examples/anthropic/custom_graph_haiku.py

@@ -5,10 +5,9 @@
 import os
 from dotenv import load_dotenv
 
-from langchain_openai import OpenAIEmbeddings
-from langchain_openai import ChatOpenAI
+from langchain_anthropic import ChatAnthropic
 from scrapegraphai.graphs import BaseGraph
-from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode
+from scrapegraphai.nodes import FetchNode, ParseNode, GenerateAnswerNode, RobotsNode
 load_dotenv()
 
 # ************************************************
@@ -19,16 +18,14 @@
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
         "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+    },
 }
 
 # ************************************************
 # Define the graph nodes
 # ************************************************
 
-llm_model = OpenAI(graph_config["llm"])
-embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key)
+llm_model = ChatAnthropic(graph_config["llm"])
 
 # define the nodes for the graph
 robot_node = RobotsNode(
@@ -43,7 +40,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
@@ -57,15 +54,6 @@
         "verbose": True,
     }
 )
-rag_node = RAGNode(
-    input="user_prompt & (parsed_doc | doc)",
-    output=["relevant_chunks"],
-    node_config={
-        "llm_model": llm_model,
-        "embedder_model": embedder,
-        "verbose": True,
-    }
-)
 generate_answer_node = GenerateAnswerNode(
     input="user_prompt & (relevant_chunks | parsed_doc | doc)",
     output=["answer"],
@@ -84,14 +72,12 @@
         robot_node,
         fetch_node,
         parse_node,
-        rag_node,
         generate_answer_node,
     ],
     edges=[
         (robot_node, fetch_node),
         (fetch_node, parse_node),
-        (parse_node, rag_node),
-        (rag_node, generate_answer_node)
+        (parse_node, generate_answer_node)
     ],
     entry_point=robot_node
 )

examples/anthropic/json_scraper_haiku.py

@@ -26,9 +26,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

examples/anthropic/json_scraper_multi_haiku.py

@@ -11,9 +11,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 FILE_NAME = "inputs/example.json"

examples/anthropic/pdf_scraper_graph_haiku.py

@@ -14,9 +14,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 source = """

examples/anthropic/pdf_scraper_multi_haiku.py

@@ -11,9 +11,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ***************

examples/anthropic/rate_limit_haiku.py

@@ -0,0 +1,48 @@
+""" 
+Basic example of scraping pipeline using SmartScraper while setting an API rate limit.
+"""
+
+import os
+from dotenv import load_dotenv
+from scrapegraphai.graphs import SmartScraperGraph
+from scrapegraphai.utils import prettify_exec_info
+
+
+# required environment variables in .env
+# ANTHROPIC_API_KEY
+load_dotenv()
+
+# ************************************************
+# Create the SmartScraperGraph instance and run it
+# ************************************************
+
+graph_config = {
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "anthropic/claude-3-haiku-20240307",
+        "rate_limit": {
+            "requests_per_second": 1
+        }
+    },
+}
+
+smart_scraper_graph = SmartScraperGraph(
+    prompt="""Don't say anything else. Output JSON only. List me all the events, with the following fields: company_name, event_name, event_start_date, event_start_time, 
+    event_end_date, event_end_time, location, event_mode, event_category, 
+    third_party_redirect, no_of_days, 
+    time_in_hours, hosted_or_attending, refreshments_type, 
+    registration_available, registration_link""",
+    # also accepts a string with the already downloaded HTML code
+    source="https://www.hmhco.com/event",
+    config=graph_config
+)
+
+result = smart_scraper_graph.run()
+print(result)
+
+# ************************************************
+# Get graph execution info
+# ************************************************
+
+graph_exec_info = smart_scraper_graph.get_execution_info()
+print(prettify_exec_info(graph_exec_info))

examples/anthropic/scrape_plain_text_haiku.py

@@ -28,9 +28,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

examples/anthropic/script_generator_haiku.py

@@ -16,9 +16,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

examples/anthropic/script_multi_generator_haiku.py

@@ -16,10 +16,9 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
-        "library": "beautifulsoup"
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
+    "library": "beautifulsoup"
 }
 
 # ************************************************

examples/anthropic/search_graph_haiku.py

@@ -15,9 +15,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

examples/anthropic/search_graph_schema_haiku.py

@@ -5,7 +5,7 @@
 import os
 from typing import List
 from dotenv import load_dotenv
-from pydantic import BaseModel, Field
+from langchain_core.pydantic_v1 import BaseModel, Field
 from scrapegraphai.graphs import SearchGraph
 
 load_dotenv()
@@ -27,8 +27,8 @@ class Dishes(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000},
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

examples/anthropic/search_link_graph_haiku.py

@@ -14,23 +14,11 @@
 
 load_dotenv()
 
-llm_model_instance = AzureChatOpenAI(
-    openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
-    azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
-)
-
-embedder_model_instance = AzureOpenAIEmbeddings(
-    azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
-    openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
-)
-
-# ************************************************
-# Create the SmartScraperGraph instance and run it
-# ************************************************
-
 graph_config = {
-    "llm": {"model_instance": llm_model_instance},
-    "embeddings": {"model_instance": embedder_model_instance}
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

examples/anthropic/smart_scraper_haiku.py

@@ -19,9 +19,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 smart_scraper_graph = SmartScraperGraph(