Merge branch 'pre/beta' into refactoring-tokenization

Author: VinciGit00

CHANGELOG.md

@@ -32,26 +32,38 @@ playwright install
 
 **Note**: it is recommended to install the library in a virtual environment to avoid conflicts with other libraries 🐱
 
-By the way if you to use not mandatory modules it is necessary to install by yourself with the following command:
+<details>
+<summary><b>Optional Dependencies</b></summary>
+Additional dependecies can be added while installing the library:
 
-### Installing "Other Language Models"
+- <b>More Language Models</b>: additional language models are installed, such as Fireworks, Groq, Anthropic, Hugging Face, and Nvidia AI Endpoints.
 
-This group allows you to use additional language models like Fireworks, Groq, Anthropic, Hugging Face, and Nvidia AI Endpoints.
+
+This group allows you to use additional language models like Fireworks, Groq, Anthropic, Together AI, Hugging Face, and Nvidia AI Endpoints.
 ```bash
 pip install scrapegraphai[other-language-models]
 
-```
-### Installing "More Semantic Options"
+- <b>Semantic Options</b>: this group includes tools for advanced semantic processing, such as Graphviz.
+
+  ```bash
+  pip install scrapegraphai[more-semantic-options]
+  ```
+
+- <b>Browsers Options</b>: this group includes additional browser management tools/services, such as Browserbase.
+
+  ```bash
+  pip install scrapegraphai[more-browser-options]
+  ```
+
+</details>
+
+
 
-This group includes tools for advanced semantic processing, such as Graphviz.
-```bash
-pip install scrapegraphai[more-semantic-options]
-```
 ### Installing "More Browser Options"
 
-This group includes additional browser management options, such as BrowserBase.
+This group includes an ocr scraper for websites
 ```bash
-pip install scrapegraphai[more-browser-options]
+pip install scrapegraphai[screenshot_scraper]
 ```
 
 ## 💻 Usage
@@ -68,7 +80,7 @@ from scrapegraphai.graphs import SmartScraperGraph
 graph_config = {
     "llm": {
         "api_key": "YOUR_OPENAI_APIKEY",
-        "model": "gpt-4o-mini",
+        "model": "openai/gpt-4o-mini",
     },
     "verbose": True,
     "headless": False,
@@ -128,6 +140,9 @@ Check out also the Docusaurus [here](https://scrapegraph-doc.onrender.com/).
 
 ## 🏆 Sponsors
 <div style="text-align: center;">
+  <a href="https://2ly.link/1zaXG">
+    <img src="https://raw.githubusercontent.com/VinciGit00/Scrapegraph-ai/main/docs/assets/browserbase_logo.png" alt="Browserbase" style="width: 10%;">
+  </a>
   <a href="https://2ly.link/1zNiz">
     <img src="https://raw.githubusercontent.com/VinciGit00/Scrapegraph-ai/main/docs/assets/serp_api_logo.png" alt="SerpAPI" style="width: 10%;">
   </a>

README.md

@@ -82,6 +82,11 @@ FAQ
 Sponsors
 ========
 
+.. image:: ../../assets/browserbase_logo.png
+   :width: 10%
+   :alt: Browserbase
+   :target: https://www.browserbase.com/
+
 .. image:: ../../assets/serp_api_logo.png
    :width: 10%
    :alt: Serp API

docs/assets/browserbase_logo.png

@@ -26,8 +26,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000},
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

docs/source/introduction/overview.rst

@@ -32,9 +32,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

examples/anthropic/csv_scraper_graph_multi_haiku.py

@@ -5,10 +5,9 @@
 import os
 from dotenv import load_dotenv
 
-from langchain_openai import OpenAIEmbeddings
-from langchain_openai import ChatOpenAI
+from langchain_anthropic import ChatAnthropic
 from scrapegraphai.graphs import BaseGraph
-from scrapegraphai.nodes import FetchNode, ParseNode, RAGNode, GenerateAnswerNode, RobotsNode
+from scrapegraphai.nodes import FetchNode, ParseNode, GenerateAnswerNode, RobotsNode
 load_dotenv()
 
 # ************************************************
@@ -19,16 +18,14 @@
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
         "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+    },
 }
 
 # ************************************************
 # Define the graph nodes
 # ************************************************
 
-llm_model = ChatOpenAI(graph_config["llm"])
-embedder = OpenAIEmbeddings(api_key=llm_model.openai_api_key)
+llm_model = ChatAnthropic(graph_config["llm"])
 
 # define the nodes for the graph
 robot_node = RobotsNode(
@@ -43,7 +40,7 @@
 
 fetch_node = FetchNode(
     input="url | local_dir",
-    output=["doc", "link_urls", "img_urls"],
+    output=["doc"],
     node_config={
         "verbose": True,
         "headless": True,
@@ -76,14 +73,12 @@
         robot_node,
         fetch_node,
         parse_node,
-        rag_node,
         generate_answer_node,
     ],
     edges=[
         (robot_node, fetch_node),
         (fetch_node, parse_node),
-        (parse_node, rag_node),
-        (rag_node, generate_answer_node)
+        (parse_node, generate_answer_node)
     ],
     entry_point=robot_node
 )

examples/anthropic/csv_scraper_haiku.py

@@ -26,9 +26,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 # ************************************************

examples/anthropic/custom_graph_haiku.py

@@ -11,9 +11,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 FILE_NAME = "inputs/example.json"

examples/anthropic/json_scraper_haiku.py

@@ -14,9 +14,8 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000
-        },
+        "model": "anthropic/claude-3-haiku-20240307",
+    },
 }
 
 source = """