refactoring of the code

Author: VinciGit00

scrapegraphai/docloaders/browser_base.py

@@ -13,8 +13,10 @@ def browser_base_fetch(api_key: str, project_id: str, link: List[str], text_cont
     - `api_key`: The API key provided by BrowserBase.
     - `project_id`: The ID of the project on BrowserBase where you want to fetch data from.
     - `link`: The URL or link that you want to fetch data from.
-    - `text_content`: A boolean flag to specify whether to return only the text content (True) or the full HTML (False).
-    - `async_mode`: A boolean flag that determines whether the function runs asynchronously (True) or synchronously (False, default).
+    - `text_content`: A boolean flag to specify whether to return only the 
+        text content (True) or the full HTML (False).
+    - `async_mode`: A boolean flag that determines whether the function runs asynchronously 
+        (True) or synchronously (False, default).
 
     It initializes a Browserbase object with the given API key and project ID, 
     then uses this object to load the specified link. 
@@ -37,8 +39,10 @@ def browser_base_fetch(api_key: str, project_id: str, link: List[str], text_cont
         api_key (str): The API key provided by BrowserBase.
         project_id (str): The ID of the project on BrowserBase where you want to fetch data from.
         link (str): The URL or link that you want to fetch data from.
-        text_content (bool): Whether to return only the text content (True) or the full HTML (False). Defaults to True.
-        async_mode (bool): Whether to run the function asynchronously (True) or synchronously (False). Defaults to False.
+        text_content (bool): Whether to return only the text content 
+        (True) or the full HTML (False). Defaults to True.
+        async_mode (bool): Whether to run the function asynchronously 
+        (True) or synchronously (False). Defaults to False.
 
     Returns:
         object: The result of the loading operation.
@@ -47,7 +51,8 @@ def browser_base_fetch(api_key: str, project_id: str, link: List[str], text_cont
     try:
         from browserbase import Browserbase
     except ImportError:
-        raise ImportError("The browserbase module is not installed. Please install it using `pip install browserbase`.")
+        raise ImportError(f"""The browserbase module is not installed. 
+                          Please install it using `pip install browserbase`.""")
 
 
     browserbase = Browserbase(api_key=api_key, project_id=project_id)

scrapegraphai/docloaders/scrape_do.py

@@ -24,17 +24,14 @@ def scrape_do_fetch(token, target_url, use_proxy=False, geoCode=None, super_prox
     """
     encoded_url = urllib.parse.quote(target_url)
     if use_proxy:
-        # Create proxy mode URL
-        proxyModeUrl = f"http://{token}:@proxy.scrape.do:8080"
+        proxy_mode_url = f"http://{token}:@proxy.scrape.do:8080"
         proxies = {
-            "http": proxyModeUrl,
-            "https": proxyModeUrl,
+            "http": proxy_mode_url,
+            "https": proxy_mode_url,
         }
-        # Add optional geoCode and super proxy parameters if provided
         params = {"geoCode": geoCode, "super": str(super_proxy).lower()} if geoCode else {}
         response = requests.get(target_url, proxies=proxies, verify=False, params=params)
     else:
-        # API Mode URL
         url = f"http://api.scrape.do?token={token}&url={encoded_url}"
         response = requests.get(url)
 

scrapegraphai/graphs/__init__.py

@@ -26,4 +26,4 @@
 from .search_link_graph import SearchLinkGraph
 from .screenshot_scraper_graph import ScreenshotScraperGraph
 from .smart_scraper_multi_concat_graph import SmartScraperMultiConcatGraph
-from .code_generator_graph import CodeGeneratorGraph
+from .code_generator_graph import CodeGeneratorGraph

scrapegraphai/graphs/abstract_graph.py

@@ -128,7 +128,7 @@ def _create_llm(self, llm_config: dict) -> object:
             if requests_per_second is not None:
                 with warnings.catch_warnings():
                     warnings.simplefilter("ignore")
-                    llm_params["rate_limiter"] =  InMemoryRateLimiter(requests_per_second=requests_per_second)
+                    llm_params["rate_limiter"] = InMemoryRateLimiter(requests_per_second=requests_per_second)
             if max_retries is not None:
                 llm_params["max_retries"] = max_retries
 

scrapegraphai/graphs/base_graph.py

@@ -45,7 +45,8 @@ class BaseGraph:
         ... )
     """
 
-    def __init__(self, nodes: list, edges: list, entry_point: str, use_burr: bool = False, burr_config: dict = None, graph_name: str = "Custom"):
+    def __init__(self, nodes: list, edges: list, entry_point: str, 
+                 use_burr: bool = False, burr_config: dict = None, graph_name: str = "Custom"):
         self.nodes = nodes
         self.raw_edges = edges
         self.edges = self._create_edges({e for e in edges})

scrapegraphai/graphs/code_generator_graph.py

@@ -51,7 +51,7 @@ class CodeGeneratorGraph(AbstractGraph):
     """
 
     def __init__(self, prompt: str, source: str, config: dict, schema: Optional[BaseModel] = None):
-        
+
         super().__init__(prompt, config, source, schema)
 
         self.input_key = "url" if source.startswith("http") else "local_dir"
@@ -63,7 +63,7 @@ def _create_graph(self) -> BaseGraph:
         Returns:
             BaseGraph: A graph instance representing the web scraping workflow.
         """
-        
+
         if self.schema is None: 
             raise KeyError("The schema is required for CodeGeneratorGraph")
 

scrapegraphai/graphs/deep_scraper_graph.py

@@ -134,7 +134,6 @@ def _create_repeated_graph(self) -> BaseGraph:
         )
 
 
-
     def _create_graph(self) -> BaseGraph:
         """
         Creates the graph of nodes representing the workflow for web scraping

scrapegraphai/graphs/omni_search_graph.py

@@ -14,7 +14,6 @@
 )
 from ..utils.copy import safe_deepcopy
 
-
 class OmniSearchGraph(AbstractGraph):
     """ 
     OmniSearchGraph is a scraping pipeline that searches the internet for answers to a given prompt.

scrapegraphai/graphs/screenshot_scraper_graph.py

@@ -47,6 +47,7 @@ def _create_graph(self) -> BaseGraph:
                 "link": self.source
             }
         )
+
         generate_answer_from_image_node = GenerateAnswerFromImageNode(
             input="screenshots",
             output=["answer"],

scrapegraphai/graphs/script_creator_graph.py

@@ -68,6 +68,7 @@ def _create_graph(self) -> BaseGraph:
                 "script_creator": True
             }
         )
+
         parse_node = ParseNode(
             input="doc",
             output=["parsed_doc"],
@@ -76,6 +77,7 @@ def _create_graph(self) -> BaseGraph:
                          "llm_model": self.llm_model
                          }
         )
+
         generate_scraper_node = GenerateScraperNode(
             input="user_prompt & (parsed_doc)",
             output=["answer"],