fixed a bug

Author: VinciGit00

scrapegraphai/graphs/script_creator_graph.py

@@ -64,8 +64,13 @@ def _create_graph(self) -> BaseGraph:
         """
 
         fetch_node = FetchNode(
-            input="url_for_script | local_dir",
+            input="url | local_dir",
             output=["doc", "link_urls", "img_urls"],
+            node_config={
+                "llm_model": self.llm_model,
+                "loader_kwargs": self.config.get("loader_kwargs", {}),
+                "script_creator": True
+            }
         )
         parse_node = ParseNode(
             input="doc",

scrapegraphai/graphs/smart_scraper_graph.py

@@ -62,7 +62,7 @@ def _create_graph(self) -> BaseGraph:
             BaseGraph: A graph instance representing the web scraping workflow.
         """
         fetch_node = FetchNode(
-            input="url_for_scraping | local_dir",
+            input="url| local_dir",
             output=["doc", "link_urls", "img_urls"],
             node_config={
                 "llm_model": self.llm_model,

scrapegraphai/nodes/fetch_node.py

@@ -62,8 +62,10 @@ def __init__(
             {} if node_config is None else node_config.get("llm_model", {})
         )
         self.force = (
-            {} if node_config is None else node_config.get("force", {})
+            {} if node_config is None else node_config.get("force", False)
         )
+        self.script_creator = node_config.get("script_creator", False)
+
 
     def execute(self, state):
         """
@@ -146,7 +148,7 @@ def execute(self, state):
 
             parsed_content = source
 
-            if  isinstance(self.llm_model, OpenAI) and self.input == "-----" or self.force:
+            if  isinstance(self.llm_model, OpenAI) and not self.script_creator or self.force and not self.script_creator:
                 parsed_content = convert_to_md(source)
 
             compressed_document = [
@@ -162,7 +164,7 @@ def execute(self, state):
 
                 parsed_content = source
 
-                if  isinstance(self.llm_model, OpenAI) and self.input == "-----" or self.force:
+                if  isinstance(self.llm_model, OpenAI) and not self.script_creator or self.force and not self.script_creator:
                     parsed_content = convert_to_md(source)
                 compressed_document = [Document(page_content=parsed_content)]
             else:
@@ -184,7 +186,7 @@ def execute(self, state):
                 raise ValueError("No HTML body content found in the document fetched by ChromiumLoader.")
             parsed_content = document[0].page_content
 
-            if  isinstance(self.llm_model, OpenAI) and self.input == "-----" or self.force:
+            if  isinstance(self.llm_model, OpenAI) and not self.script_creator or self.force and not self.script_creator:
                 parsed_content = convert_to_md(document[0].page_content)
 
             compressed_document = [