From 0c476a4a7bbbec3883f505cd47bcffdcd2d9e5fd Mon Sep 17 00:00:00 2001 From: Marco Vinciguerra Date: Fri, 30 May 2025 14:25:24 +0200 Subject: [PATCH 1/5] feat: add grok integration --- scrapegraphai/graphs/abstract_graph.py | 7 ++++++- scrapegraphai/helpers/models_tokens.py | 5 ++++- scrapegraphai/models/__init__.py | 3 ++- scrapegraphai/models/xai.py | 23 +++++++++++++++++++++++ 4 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 scrapegraphai/models/xai.py diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py index c42e4daf..90f6efc2 100644 --- a/scrapegraphai/graphs/abstract_graph.py +++ b/scrapegraphai/graphs/abstract_graph.py @@ -13,7 +13,7 @@ from pydantic import BaseModel from ..helpers import models_tokens -from ..models import CLoD, DeepSeek, OneApi +from ..models import CLoD, DeepSeek, OneApi, XAI from ..utils.logging import set_verbosity_info, set_verbosity_warning @@ -163,6 +163,7 @@ def _create_llm(self, llm_config: dict) -> object: "fireworks", "clod", "togetherai", + "xai", } if "/" in llm_params["model"]: @@ -217,6 +218,7 @@ def _create_llm(self, llm_config: dict) -> object: "deepseek", "togetherai", "clod", + "xai", }: if llm_params["model_provider"] == "bedrock": llm_params["model_kwargs"] = { @@ -242,6 +244,9 @@ def _create_llm(self, llm_config: dict) -> object: elif model_provider == "oneapi": return OneApi(**llm_params) + elif model_provider == "xai": + return XAI(**llm_params) + elif model_provider == "togetherai": try: from langchain_together import ChatTogether diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py index ec8e935d..4cd9fe6e 100644 --- a/scrapegraphai/helpers/models_tokens.py +++ b/scrapegraphai/helpers/models_tokens.py @@ -150,7 +150,7 @@ "llama3-70b-8192": 8192, "mixtral-8x7b-32768": 32768, "gemma-7b-it": 8192, - "claude-3-haiku-20240307'": 8192, + "claude-3-haiku-20240307": 8192, }, "toghetherai": { "meta-llama/Meta-Llama-3.1-8B-Instruct-Turbo": 128000, @@ -303,4 +303,7 @@ "grok-2-latest": 128000, }, "togetherai": {"Meta-Llama-3.1-70B-Instruct-Turbo": 128000}, + "xai": { + "grok-1": 8192 + }, } diff --git a/scrapegraphai/models/__init__.py b/scrapegraphai/models/__init__.py index bb736a6b..0031f9a1 100644 --- a/scrapegraphai/models/__init__.py +++ b/scrapegraphai/models/__init__.py @@ -7,5 +7,6 @@ from .oneapi import OneApi from .openai_itt import OpenAIImageToText from .openai_tts import OpenAITextToSpeech +from .xai import XAI -__all__ = ["DeepSeek", "OneApi", "OpenAIImageToText", "OpenAITextToSpeech", "CLoD"] +__all__ = ["DeepSeek", "OneApi", "OpenAIImageToText", "OpenAITextToSpeech", "CLoD", "XAI"] diff --git a/scrapegraphai/models/xai.py b/scrapegraphai/models/xai.py new file mode 100644 index 00000000..405f88b6 --- /dev/null +++ b/scrapegraphai/models/xai.py @@ -0,0 +1,23 @@ +""" +xAI Grok Module +""" +from langchain_groq import ChatGroq as LangchainChatGroq + +class XAI(LangchainChatGroq): + """ + Wrapper for the ChatGroq class from langchain_groq, for use with xAI models. + Handles API key mapping from generic 'api_key' to 'groq_api_key' and + maps 'model' to 'model_name'. + + Args: + llm_config (dict): Configuration parameters for the language model. + """ + + def __init__(self, **llm_config): + if "api_key" in llm_config and "groq_api_key" not in llm_config: + llm_config["groq_api_key"] = llm_config.pop("api_key") + + if "model" in llm_config and "model_name" not in llm_config: + llm_config["model_name"] = llm_config.pop("model") + + super().__init__(**llm_config) \ No newline at end of file From 7adb0f1df1efc4e6ada1134f6e53e4d6b072a608 Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 30 May 2025 12:26:44 +0000 Subject: [PATCH 2/5] ci(release): 1.52.0-beta.1 [skip ci] ## [1.52.0-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.51.0...v1.52.0-beta.1) (2025-05-30) ### Features * add grok integration ([0c476a4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0c476a4a7bbbec3883f505cd47bcffdcd2d9e5fd)) ### CI * **release:** 1.50.0-beta.1 [skip ci] ([470ed48](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/470ed4893f8acaf53cb283497cb1fc6e263cc790)) --- CHANGELOG.md | 12 ++++++++++++ pyproject.toml | 2 +- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 18875859..3f608666 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,15 @@ +## [1.52.0-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.51.0...v1.52.0-beta.1) (2025-05-30) + + +### Features + +* add grok integration ([0c476a4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0c476a4a7bbbec3883f505cd47bcffdcd2d9e5fd)) + + +### CI + +* **release:** 1.50.0-beta.1 [skip ci] ([470ed48](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/470ed4893f8acaf53cb283497cb1fc6e263cc790)) + ## [1.50.0-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.49.0...v1.50.0-beta.1) (2025-04-29) diff --git a/pyproject.toml b/pyproject.toml index cb5144f6..add853f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "scrapegraphai" -version = "1.50.0b1" +version = "1.52.0b1" description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." authors = [ From 3f1827274c60a2729233577666d2fa446c48c4ba Mon Sep 17 00:00:00 2001 From: Vinh Thieu Date: Sat, 31 May 2025 00:13:44 +0700 Subject: [PATCH 3/5] fix: grok integration and add new grok models --- scrapegraphai/helpers/models_tokens.py | 4 +++- scrapegraphai/models/xai.py | 19 +++++++++---------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/scrapegraphai/helpers/models_tokens.py b/scrapegraphai/helpers/models_tokens.py index 4cd9fe6e..a0258fa9 100644 --- a/scrapegraphai/helpers/models_tokens.py +++ b/scrapegraphai/helpers/models_tokens.py @@ -304,6 +304,8 @@ }, "togetherai": {"Meta-Llama-3.1-70B-Instruct-Turbo": 128000}, "xai": { - "grok-1": 8192 + "grok-1": 8192, + "grok-3": 128000, + "grok-3-mini": 128000, }, } diff --git a/scrapegraphai/models/xai.py b/scrapegraphai/models/xai.py index 405f88b6..86969483 100644 --- a/scrapegraphai/models/xai.py +++ b/scrapegraphai/models/xai.py @@ -1,23 +1,22 @@ """ xAI Grok Module """ -from langchain_groq import ChatGroq as LangchainChatGroq +from langchain_openai import ChatOpenAI -class XAI(LangchainChatGroq): + +class XAI(ChatOpenAI): """ - Wrapper for the ChatGroq class from langchain_groq, for use with xAI models. - Handles API key mapping from generic 'api_key' to 'groq_api_key' and - maps 'model' to 'model_name'. + A wrapper for the ChatOpenAI class (xAI uses an OpenAI-compatible API) that + provides default configuration and could be extended with additional methods + if needed. Args: llm_config (dict): Configuration parameters for the language model. """ def __init__(self, **llm_config): - if "api_key" in llm_config and "groq_api_key" not in llm_config: - llm_config["groq_api_key"] = llm_config.pop("api_key") - - if "model" in llm_config and "model_name" not in llm_config: - llm_config["model_name"] = llm_config.pop("model") + if "api_key" in llm_config: + llm_config["openai_api_key"] = llm_config.pop("api_key") + llm_config["openai_api_base"] = "https://api.x.ai/v1" super().__init__(**llm_config) \ No newline at end of file From 386b46a8692c8c18000bb071fc8f312adc3ad05e Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Wed, 4 Jun 2025 08:15:53 +0000 Subject: [PATCH 4/5] ci(release): 1.52.0-beta.2 [skip ci] ## [1.52.0-beta.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.52.0-beta.1...v1.52.0-beta.2) (2025-06-04) ### Bug Fixes * grok integration and add new grok models ([3f18272](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3f1827274c60a2729233577666d2fa446c48c4ba)) --- CHANGELOG.md | 7 +++++++ pyproject.toml | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f608666..342f6e5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,10 @@ +## [1.52.0-beta.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.52.0-beta.1...v1.52.0-beta.2) (2025-06-04) + + +### Bug Fixes + +* grok integration and add new grok models ([3f18272](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3f1827274c60a2729233577666d2fa446c48c4ba)) + ## [1.52.0-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.51.0...v1.52.0-beta.1) (2025-05-30) diff --git a/pyproject.toml b/pyproject.toml index add853f4..d6627bd3 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "scrapegraphai" -version = "1.52.0b1" +version = "1.52.0b2" description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." authors = [ From 77d44321a1d41e10ac6aa13b526a49e718bd7c5d Mon Sep 17 00:00:00 2001 From: semantic-release-bot Date: Fri, 6 Jun 2025 10:47:41 +0000 Subject: [PATCH 5/5] ci(release): 1.54.0-beta.1 [skip ci] ## [1.54.0-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.53.0...v1.54.0-beta.1) (2025-06-06) ### Features * add grok integration ([0c476a4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0c476a4a7bbbec3883f505cd47bcffdcd2d9e5fd)) ### Bug Fixes * grok integration and add new grok models ([3f18272](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3f1827274c60a2729233577666d2fa446c48c4ba)) ### chore * enhanced a readme ([68bb34c](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/68bb34cc5e63b8a1d5acc61b9b61f9ea716a2a51)) ### CI * **release:** 1.52.0-beta.1 [skip ci] ([7adb0f1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/7adb0f1df1efc4e6ada1134f6e53e4d6b072a608)) * **release:** 1.52.0-beta.2 [skip ci] ([386b46a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/386b46a8692c8c18000bb071fc8f312adc3ad05e)) --- CHANGELOG.md | 23 +++++++++++++++++++++++ pyproject.toml | 2 +- 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 342f6e5d..e4bd45e3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,26 @@ +## [1.54.0-beta.1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.53.0...v1.54.0-beta.1) (2025-06-06) + + +### Features + +* add grok integration ([0c476a4](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0c476a4a7bbbec3883f505cd47bcffdcd2d9e5fd)) + + +### Bug Fixes + +* grok integration and add new grok models ([3f18272](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3f1827274c60a2729233577666d2fa446c48c4ba)) + + +### chore + +* enhanced a readme ([68bb34c](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/68bb34cc5e63b8a1d5acc61b9b61f9ea716a2a51)) + + +### CI + +* **release:** 1.52.0-beta.1 [skip ci] ([7adb0f1](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/7adb0f1df1efc4e6ada1134f6e53e4d6b072a608)) +* **release:** 1.52.0-beta.2 [skip ci] ([386b46a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/386b46a8692c8c18000bb071fc8f312adc3ad05e)) + ## [1.52.0-beta.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.52.0-beta.1...v1.52.0-beta.2) (2025-06-04) diff --git a/pyproject.toml b/pyproject.toml index 3d502e25..80d76fa7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [project] name = "scrapegraphai" -version = "1.53.0" +version = "1.54.0b1" description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines." authors = [