Merge pull request #417 from ScrapeGraphAI/md_scraper_integration

feat: add integrations for markdown files

Author: VinciGit00

README.md

@@ -12,7 +12,7 @@
 [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg?style=for-the-badge)](https://opensource.org/licenses/MIT)
 [![](https://dcbadge.vercel.app/api/server/gkxQDAjfeX)](https://discord.gg/gkxQDAjfeX)
 
-ScrapeGraphAI is a *web scraping* python library that uses LLM and direct graph logic to create scraping pipelines for websites and local documents (XML, HTML, JSON, etc.).
+ScrapeGraphAI is a *web scraping* python library that uses LLM and direct graph logic to create scraping pipelines for websites and local documents (XML, HTML, JSON, Markdown, etc.).
 
 Just say which information you want to extract and the library will do it for you!
 

scrapegraphai/graphs/markdown_scraper_graph.py

@@ -78,6 +78,7 @@ def _create_graph(self) -> BaseGraph:
                 "llm_model": self.llm_model,
                 "additional_info": self.config.get("additional_info"),
                 "schema": self.schema,
+                "is_md_scraper": True
             }
         )
 

scrapegraphai/nodes/generate_answer_node.py

@@ -53,6 +53,9 @@ def __init__(
         self.script_creator = (
             False if node_config is None else node_config.get("script_creator", False)
         )
+        self.is_md_scraper = (
+            False if node_config is None else node_config.get("is_md_scraper", False)
+        )
 
         self.additional_info = node_config.get("additional_info")
 
@@ -90,7 +93,7 @@ def execute(self, state: dict) -> dict:
 
         format_instructions = output_parser.get_format_instructions()
 
-        if  isinstance(self.llm_model, OpenAI) and not self.script_creator or self.force and not self.script_creator:
+        if  isinstance(self.llm_model, OpenAI) and not self.script_creator or self.force and not self.script_creator or self.is_md_scraper:
             template_no_chunks_prompt = template_no_chunks_md
             template_chunks_prompt = template_chunks_md
             template_merge_prompt = template_merge_md