Projects
+ +
+
+
+
+
+
+
+
+
+
+
+
+ Hello World!
" >>> remover(html_content) 'Title: Example, Body:Hello World!
' - This function is particularly useful for preparing HTML content for environments where bandwidth usage needs to be minimized. + This function is particularly useful for preparing HTML content for + environments where bandwidth usage needs to be minimized. """ soup = BeautifulSoup(html_content, 'html.parser') @@ -55,4 +58,5 @@ def cleanup_html(html_content: str, base_url: str) -> str: return title, minimized_body, link_urls, image_urls else: - raise ValueError(f"No HTML body content found, please try setting the 'headless' flag to False in the graph configuration. HTML content: {html_content}") + raise ValueError(f"""No HTML body content found, please try setting the 'headless' + flag to False in the graph configuration. HTML content: {html_content}""") diff --git a/scrapegraphai/utils/convert_to_csv.py b/scrapegraphai/utils/convert_to_csv.py index be001d06..44897c7c 100644 --- a/scrapegraphai/utils/convert_to_csv.py +++ b/scrapegraphai/utils/convert_to_csv.py @@ -5,7 +5,6 @@ import sys import pandas as pd - def convert_to_csv(data: dict, filename: str, position: str = None) -> None: """ Converts a dictionary to a CSV file and saves it at a specified location. diff --git a/scrapegraphai/utils/convert_to_json.py b/scrapegraphai/utils/convert_to_json.py index 7cf12c53..57618fc1 100644 --- a/scrapegraphai/utils/convert_to_json.py +++ b/scrapegraphai/utils/convert_to_json.py @@ -5,7 +5,6 @@ import os import sys - def convert_to_json(data: dict, filename: str, position: str = None) -> None: """ Converts a dictionary to a JSON file and saves it at a specified location. diff --git a/scrapegraphai/utils/convert_to_md.py b/scrapegraphai/utils/convert_to_md.py index 4c22d35b..123f3457 100644 --- a/scrapegraphai/utils/convert_to_md.py +++ b/scrapegraphai/utils/convert_to_md.py @@ -1,5 +1,5 @@ """ -convert_to_md modul +convert_to_md module """ from urllib.parse import urlparse import html2text @@ -27,5 +27,5 @@ def convert_to_md(html: str, url: str = None) -> str: parsed_url = urlparse(url) domain = f"{parsed_url.scheme}://{parsed_url.netloc}" h.baseurl = domain - + return h.handle(html) diff --git a/scrapegraphai/utils/logging.py b/scrapegraphai/utils/logging.py index afb63c52..b40c2cd8 100644 --- a/scrapegraphai/utils/logging.py +++ b/scrapegraphai/utils/logging.py @@ -17,7 +17,6 @@ _semaphore = threading.Lock() - def _get_library_root_logger() -> logging.Logger: return logging.getLogger(_library_name) diff --git a/scrapegraphai/utils/prettify_exec_info.py b/scrapegraphai/utils/prettify_exec_info.py index 6bda73c6..8cfef81a 100644 --- a/scrapegraphai/utils/prettify_exec_info.py +++ b/scrapegraphai/utils/prettify_exec_info.py @@ -1,7 +1,6 @@ """ Prettify the execution information of the graph. """ - import pandas as pd diff --git a/scrapegraphai/utils/proxy_rotation.py b/scrapegraphai/utils/proxy_rotation.py index 6f6019e9..586e640e 100644 --- a/scrapegraphai/utils/proxy_rotation.py +++ b/scrapegraphai/utils/proxy_rotation.py @@ -10,7 +10,6 @@ from fp.errors import FreeProxyException from fp.fp import FreeProxy - class ProxyBrokerCriteria(TypedDict, total=False): """proxy broker criteria""" diff --git a/scrapegraphai/utils/save_audio_from_bytes.py b/scrapegraphai/utils/save_audio_from_bytes.py index 3027e4e8..2bad3106 100644 --- a/scrapegraphai/utils/save_audio_from_bytes.py +++ b/scrapegraphai/utils/save_audio_from_bytes.py @@ -11,7 +11,8 @@ def save_audio_from_bytes(byte_response: bytes, output_path: Union[str, Path]) - Args: byte_response (bytes): The byte array containing audio data. - output_path (Union[str, Path]): The destination file path where the audio file will be saved. + output_path (Union[str, Path]): The destination + file path where the audio file will be saved. Example: >>> save_audio_from_bytes(b'audio data', 'path/to/audio.mp3') diff --git a/scrapegraphai/utils/sys_dynamic_import.py b/scrapegraphai/utils/sys_dynamic_import.py index 19b0d29a..4d1511a2 100644 --- a/scrapegraphai/utils/sys_dynamic_import.py +++ b/scrapegraphai/utils/sys_dynamic_import.py @@ -10,7 +10,6 @@ if typing.TYPE_CHECKING: import types - def srcfile_import(modpath: str, modname: str) -> "types.ModuleType": """imports a python module from its srcfile diff --git a/scrapegraphai/utils/tokenizer_openai.py b/scrapegraphai/utils/tokenizer_openai.py new file mode 100644 index 00000000..c4fb2bbd --- /dev/null +++ b/scrapegraphai/utils/tokenizer_openai.py @@ -0,0 +1,10 @@ +""" +Module for calculting the token_for_openai +""" +import tiktoken + +def num_tokens_openai(string: str) -> int: + """Returns the number of tokens in a text string.""" + encoding = tiktoken.get_encoding("cl100k_base") + num_tokens = len(encoding.encode(string)) + return num_tokens diff --git a/tests/graphs/scrape_plain_text_llama3_test.py b/tests/graphs/scrape_plain_text_llama3.1_test.py similarity index 86% rename from tests/graphs/scrape_plain_text_llama3_test.py rename to tests/graphs/scrape_plain_text_llama3.1_test.py index 93045163..6659c692 100644 --- a/tests/graphs/scrape_plain_text_llama3_test.py +++ b/tests/graphs/scrape_plain_text_llama3.1_test.py @@ -26,15 +26,10 @@ def graph_config(): """ return { "llm": { - "model": "ollama/llama3", + "model": "ollama/llama3.1", "temperature": 0, "format": "json", "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", } } diff --git a/tests/graphs/scrape_plain_text_mistral_test.py b/tests/graphs/scrape_plain_text_mistral_test.py index b887161c..888999ab 100644 --- a/tests/graphs/scrape_plain_text_mistral_test.py +++ b/tests/graphs/scrape_plain_text_mistral_test.py @@ -30,11 +30,6 @@ def graph_config(): "temperature": 0, "format": "json", "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", } } diff --git a/tests/graphs/scrape_xml_ollama_test.py b/tests/graphs/scrape_xml_ollama_test.py index 04494543..2bc38a59 100644 --- a/tests/graphs/scrape_xml_ollama_test.py +++ b/tests/graphs/scrape_xml_ollama_test.py @@ -32,11 +32,6 @@ def graph_config(): "temperature": 0, "format": "json", "base_url": "http://localhost:11434", - }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", } } diff --git a/tests/graphs/script_generator_test.py b/tests/graphs/script_generator_test.py index bf5ada42..7bcfeea7 100644 --- a/tests/graphs/script_generator_test.py +++ b/tests/graphs/script_generator_test.py @@ -18,11 +18,6 @@ def graph_config(): "base_url": "http://localhost:11434", "library": "beautifulsoup", }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - "base_url": "http://localhost:11434", - }, "library": "beautifulsoup" } diff --git a/tests/graphs/search_link_ollama.py b/tests/graphs/search_link_ollama.py index 3b41f699..530ad2a6 100644 --- a/tests/graphs/search_link_ollama.py +++ b/tests/graphs/search_link_ollama.py @@ -4,14 +4,10 @@ def test_smart_scraper_pipeline(): graph_config = { "llm": { - "model": "ollama/llama3", + "model": "ollama/llama3.1", "temperature": 0, "format": "json", }, - "embeddings": { - "model": "ollama/nomic-embed-text", - "temperature": 0, - }, "verbose": True, "headless": False } diff --git a/tests/graphs/smart_scraper_ernie_test.py b/tests/graphs/smart_scraper_ernie_test.py index 5efd8d0b..1da35790 100644 --- a/tests/graphs/smart_scraper_ernie_test.py +++ b/tests/graphs/smart_scraper_ernie_test.py @@ -16,11 +16,6 @@ def graph_config(): "ernie_client_id": "
