Pre/beta

CHANGELOG.md

@@ -1,3 +1,5 @@
+## [1.10.0-beta.7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.6...v1.10.0-beta.7) (2024-07-23)
+
 ## [1.11.3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.11.2...v1.11.3) (2024-07-25)
 
 
@@ -10,6 +12,7 @@
 
 * prev version ([5c08eea](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/5c08eea189d7ede6f29399a67d897aa3b3f6a7b0))
 
+
 ## [1.11.2](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.11.1...v1.11.2) (2024-07-23)
 
 
@@ -113,8 +116,11 @@
 
 
 
+
 ### Features
 
+* add nvidia connection ([fc0dadb](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fc0dadb8f812dfd636dec856921a971b58695ce3))
+
 
 * add new toml ([fcb3220](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fcb3220868e7ef1127a7a47f40d0379be282e6eb))
 
@@ -133,6 +139,24 @@
 
 ### chore
 
+* **dependecies:** add script to auto-update requirements ([3289c7b](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/3289c7bf5ec58ac3d04e9e5e8e654af9abcee228))
+* **ci:** set up workflow for requirements auto-update ([295fc28](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/295fc28ceb02c78198f7fbe678352503b3259b6b))
+* update requirements.txt ([c7bac98](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/c7bac98d2e79e5ab98fa65d7efa858a2cdda1622))
+
+## [1.10.0-beta.6](https://github.com/ScrapeGraphAI/Scrapegraph-ai/compare/v1.10.0-beta.5...v1.10.0-beta.6) (2024-07-22)
+
+
+### Features
+
+* add new toml ([fcb3220](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/fcb3220868e7ef1127a7a47f40d0379be282e6eb))
+* add gpt4o omni ([431edb7](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/431edb7bb2504f4c1335c3ae3ce2f91867fa7222))
+* add searchngx integration ([5c92186](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/5c9218608140bf694fbfd96aa90276bc438bb475))
+* refactoring_to_md function ([602dd00](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/602dd00209ee1d72a1223fc4793759450921fcf9))
+
+
+
+
+### chore
 
 * **pyproject:** upgrade dependencies ([0425124](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/0425124c570f765b98fcf67ba6649f4f9fe76b15))
 * correct search engine name ([7ba2f6a](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/7ba2f6ae0b9d2e9336e973e1f57ab8355c739e27))
@@ -141,11 +165,13 @@
 * upgrade tiktoken ([7314bc3](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/7314bc383068db590662bf7e512f799529308991))
 
 
+
 ### Docs
 
 * **gpt-4o-mini:** added new gpt, fixed chromium lazy loading, ([99dc849](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/99dc8497d85289759286a973e4aecc3f924d3ada))
 
 
+
 ### CI
 
 * **release:** 1.10.0-beta.1 [skip ci] ([8f619de](https://github.com/ScrapeGraphAI/Scrapegraph-ai/commit/8f619de23540216934b53bcf3426702e56c48f31))
@@ -486,7 +512,7 @@
 * **release:** 1.6.1 [skip ci] ([44fbd71](https://github.com/VinciGit00/Scrapegraph-ai/commit/44fbd71742a57a4b10f22ed33781bb67aa77e58d))
 
 ## [1.6.1](https://github.com/VinciGit00/Scrapegraph-ai/compare/v1.6.0...v1.6.1) (2024-06-15)
-=======
+
 
 
 ### Bug Fixes

pyproject.toml

@@ -1,23 +1,20 @@
 [project]
 name = "scrapegraphai"
-
-version = "1.11.3"
-
-
-
+version = "1.12.1b1"
 description = "A web scraping library based on LangChain which uses LLM and direct graph logic to create scraping pipelines."
+
 authors = [
     { name = "Marco Vinciguerra", email = "mvincig11@gmail.com" },
     { name = "Marco Perini", email = "perinim.98@gmail.com" },
     { name = "Lorenzo Padoan", email = "lorenzo.padoan977@gmail.com" }
 ]
+
 dependencies = [
     "langchain>=0.2.10",
-
     "langchain-fireworks>=0.1.3",
     "langchain_community>=0.2.9",
     "langchain-google-genai>=1.0.7",
-    "langchain-google-vertexai",
+    "langchain-google-vertexai>=1.0.7",
     "langchain-openai>=0.1.17",
     "langchain-groq>=0.1.3",
     "langchain-aws>=0.1.3",

requirements-dev.lock

@@ -12,6 +12,7 @@ aiofiles==24.1.0
     # via burr
 aiohttp==3.9.5
     # via langchain
+    # via langchain-community
     # via langchain-fireworks
     # via langchain-nvidia-ai-endpoints
 aiosignal==1.3.1
@@ -179,6 +180,7 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
+    # via sqlalchemy
 groq==0.9.0
     # via langchain-groq
 grpc-google-iam-v1==0.13.1
@@ -210,7 +212,7 @@ httpx==0.27.0
     # via openai
 httpx-sse==0.4.0
     # via fireworks-ai
-huggingface-hub==0.24.0
+huggingface-hub==0.24.1
     # via tokenizers
 idna==3.7
     # via anyio
@@ -220,7 +222,7 @@ idna==3.7
     # via yarl
 imagesize==1.4.1
     # via sphinx
-importlib-metadata==8.0.0
+importlib-metadata==8.1.0
     # via sphinx
 importlib-resources==6.4.0
     # via matplotlib
@@ -249,19 +251,20 @@ jsonschema-specifications==2023.12.1
     # via jsonschema
 kiwisolver==1.4.5
     # via matplotlib
-langchain==0.2.10
+langchain==0.2.11
     # via langchain-community
     # via scrapegraphai
 langchain-anthropic==0.1.20
     # via scrapegraphai
 langchain-aws==0.1.12
     # via scrapegraphai
-langchain-community==0.2.9
+langchain-community==0.2.10
     # via scrapegraphai
-langchain-core==0.2.22
+langchain-core==0.2.23
     # via langchain
     # via langchain-anthropic
     # via langchain-aws
+    # via langchain-community
     # via langchain-fireworks
     # via langchain-google-genai
     # via langchain-google-vertexai
@@ -277,14 +280,15 @@ langchain-google-vertexai==1.0.7
     # via scrapegraphai
 langchain-groq==0.1.6
     # via scrapegraphai
-langchain-nvidia-ai-endpoints==0.1.6
+langchain-nvidia-ai-endpoints==0.1.7
     # via scrapegraphai
 langchain-openai==0.1.17
     # via scrapegraphai
 langchain-text-splitters==0.2.2
     # via langchain
 langsmith==0.1.93
     # via langchain
+    # via langchain-community
     # via langchain-core
 loguru==0.7.2
     # via burr
@@ -319,6 +323,7 @@ numpy==1.26.4
     # via faiss-cpu
     # via langchain
     # via langchain-aws
+    # via langchain-community
     # via matplotlib
     # via pandas
     # via pyarrow
@@ -339,6 +344,7 @@ packaging==24.1
     # via google-cloud-bigquery
     # via huggingface-hub
     # via langchain-core
+    # via marshmallow
     # via matplotlib
     # via pytest
     # via sphinx
@@ -355,7 +361,7 @@ pillow==10.4.0
     # via streamlit
 platformdirs==4.2.2
     # via pylint
-playwright==1.45.0
+playwright==1.45.1
     # via scrapegraphai
     # via undetected-playwright
 pluggy==1.5.0
@@ -429,6 +435,7 @@ pytz==2024.1
 pyyaml==6.0.1
     # via huggingface-hub
     # via langchain
+    # via langchain-community
     # via langchain-core
     # via uvicorn
 referencing==0.35.1
@@ -444,6 +451,7 @@ requests==2.32.3
     # via google-cloud-storage
     # via huggingface-hub
     # via langchain
+    # via langchain-community
     # via langchain-fireworks
     # via langsmith
     # via sphinx
@@ -501,12 +509,14 @@ sphinxcontrib-serializinghtml==1.1.10
     # via sphinx
 sqlalchemy==2.0.31
     # via langchain
+    # via langchain-community
 starlette==0.37.2
     # via fastapi
 streamlit==1.36.0
     # via burr
 tenacity==8.5.0
     # via langchain
+    # via langchain-community
     # via langchain-core
     # via streamlit
 tiktoken==0.7.0
@@ -557,6 +567,7 @@ typing-extensions==4.12.2
     # via typing-inspect
     # via uvicorn
 typing-inspect==0.9.0
+    # via dataclasses-json
     # via sf-hamilton
 tzdata==2024.1
     # via pandas

requirements.lock

@@ -10,6 +10,7 @@
 -e file:.
 aiohttp==3.9.5
     # via langchain
+    # via langchain-community
     # via langchain-fireworks
     # via langchain-nvidia-ai-endpoints
 aiosignal==1.3.1
@@ -127,6 +128,7 @@ graphviz==0.20.3
     # via scrapegraphai
 greenlet==3.0.3
     # via playwright
+    # via sqlalchemy
 groq==0.9.0
     # via langchain-groq
 grpc-google-iam-v1==0.13.1
@@ -154,7 +156,7 @@ httpx==0.27.0
     # via openai
 httpx-sse==0.4.0
     # via fireworks-ai
-huggingface-hub==0.24.0
+huggingface-hub==0.24.1
     # via tokenizers
 idna==3.7
     # via anyio
@@ -170,19 +172,20 @@ jsonpatch==1.33
     # via langchain-core
 jsonpointer==3.0.0
     # via jsonpatch
-langchain==0.2.10
+langchain==0.2.11
     # via langchain-community
     # via scrapegraphai
 langchain-anthropic==0.1.20
     # via scrapegraphai
 langchain-aws==0.1.12
     # via scrapegraphai
-langchain-community==0.2.9
+langchain-community==0.2.10
     # via scrapegraphai
-langchain-core==0.2.22
+langchain-core==0.2.23
     # via langchain
     # via langchain-anthropic
     # via langchain-aws
+    # via langchain-community
     # via langchain-fireworks
     # via langchain-google-genai
     # via langchain-google-vertexai
@@ -198,14 +201,15 @@ langchain-google-vertexai==1.0.7
     # via scrapegraphai
 langchain-groq==0.1.6
     # via scrapegraphai
-langchain-nvidia-ai-endpoints==0.1.6
+langchain-nvidia-ai-endpoints==0.1.7
     # via scrapegraphai
 langchain-openai==0.1.17
     # via scrapegraphai
 langchain-text-splitters==0.2.2
     # via langchain
 langsmith==0.1.93
     # via langchain
+    # via langchain-community
     # via langchain-core
 lxml==5.2.2
     # via free-proxy
@@ -226,6 +230,7 @@ numpy==1.26.4
     # via faiss-cpu
     # via langchain
     # via langchain-aws
+    # via langchain-community
     # via pandas
     # via shapely
 openai==1.37.0
@@ -239,12 +244,13 @@ packaging==24.1
     # via google-cloud-bigquery
     # via huggingface-hub
     # via langchain-core
+    # via marshmallow
 pandas==2.2.2
     # via scrapegraphai
 pillow==10.4.0
     # via fireworks-ai
     # via langchain-nvidia-ai-endpoints
-playwright==1.45.0
+playwright==1.45.1
     # via scrapegraphai
     # via undetected-playwright
 proto-plus==1.24.0
@@ -296,6 +302,7 @@ pytz==2024.1
 pyyaml==6.0.1
     # via huggingface-hub
     # via langchain
+    # via langchain-community
     # via langchain-core
 regex==2024.5.15
     # via tiktoken
@@ -306,6 +313,7 @@ requests==2.32.3
     # via google-cloud-storage
     # via huggingface-hub
     # via langchain
+    # via langchain-community
     # via langchain-fireworks
     # via langsmith
     # via tiktoken
@@ -332,6 +340,7 @@ sqlalchemy==2.0.31
     # via langchain-community
 tenacity==8.5.0
     # via langchain
+    # via langchain-community
     # via langchain-core
 tiktoken==0.7.0
     # via langchain-openai
@@ -356,6 +365,9 @@ typing-extensions==4.12.2
     # via pydantic-core
     # via pyee
     # via sqlalchemy
+    # via typing-inspect
+typing-inspect==0.9.0
+    # via dataclasses-json
 tzdata==2024.1
     # via pandas
 undetected-playwright==0.3.0

requirements.txt

@@ -1,8 +1,8 @@
 langchain>=0.2.10
+langchain-fireworks>=0.1.3
 langchain_community>=0.2.9
 langchain-google-genai>=1.0.7
-langchain-fireworks>=0.1.3
-langchain-google-vertexai
+langchain-google-vertexai>=1.0.7
 langchain-openai>=0.1.17
 langchain-groq>=0.1.3
 langchain-aws>=0.1.3
@@ -22,4 +22,3 @@ playwright>=1.43.0
 google>=3.0.0
 undetected-playwright>=0.3.0
 semchunk>=1.0.1
-