Skip to content

Commit cb02740

Browse files
VinciGit00VinciGit00
committed
refactoring of the examples
1 parent b6cc3e3 commit cb02740

17 files changed

+101
-390
lines changed

‎examples/azure/csv_scraper_azure.py

Lines changed: 7 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55
import os
66
from dotenv import load_dotenv
77
import pandas as pd
8-
from langchain_openai import AzureChatOpenAI
9-
from langchain_openai import AzureOpenAIEmbeddings
108
from scrapegraphai.graphs import CSVScraperGraph
119
from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
1210
load_dotenv()
@@ -24,25 +22,15 @@
2422
# ************************************************
2523
# Define the configuration for the graph
2624
# ************************************************
27-
28-
llm_model_instance = AzureChatOpenAI(
29-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
30-
azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
31-
)
32-
33-
embedder_model_instance = AzureOpenAIEmbeddings(
34-
azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
35-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
36-
)
37-
38-
# ************************************************
39-
# Create the JSONScraperGraph instance and run it
40-
# ************************************************
41-
4225
graph_config = {
43-
"llm": {"model_instance": llm_model_instance},
44-
"embeddings": {"model_instance": embedder_model_instance}
26+
"llm": {
27+
"api_key": os.environ["AZURE_OPENAI_KEY"],
28+
"model": "azure/gpt-3.5-turbo",
29+
},
30+
"verbose": True,
31+
"headless": False
4532
}
33+
4634
# ************************************************
4735
# Create the CSVScraperGraph instance and run it
4836
# ************************************************

‎examples/azure/csv_scraper_graph_multi_azure.py

Lines changed: 7 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -6,8 +6,6 @@
66
from dotenv import load_dotenv
77
import pandas as pd
88
from scrapegraphai.graphs import CSVScraperMultiGraph
9-
from langchain_openai import AzureChatOpenAI
10-
from langchain_openai import AzureOpenAIEmbeddings
119
from scrapegraphai.utils import convert_to_csv, convert_to_json, prettify_exec_info
1210

1311
load_dotenv()
@@ -24,19 +22,15 @@
2422
# ************************************************
2523
# Define the configuration for the graph
2624
# ************************************************
27-
llm_model_instance = AzureChatOpenAI(
28-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
29-
azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
30-
)
31-
32-
embedder_model_instance = AzureOpenAIEmbeddings(
33-
azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
34-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
35-
)
3625
graph_config = {
37-
"llm": {"model_instance": llm_model_instance},
38-
"embeddings": {"model_instance": embedder_model_instance}
26+
"llm": {
27+
"api_key": os.environ["AZURE_OPENAI_KEY"],
28+
"model": "azure/gpt-3.5-turbo",
29+
},
30+
"verbose": True,
31+
"headless": False
3932
}
33+
4034
# ************************************************
4135
# Create the CSVScraperMultiGraph instance and run it
4236
# ************************************************

‎examples/azure/custom_graph_azure.py

Lines changed: 0 additions & 117 deletions
This file was deleted.

‎examples/azure/json_scraper_azure.py

Lines changed: 6 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -4,23 +4,11 @@
44

55
import os
66
from dotenv import load_dotenv
7-
from langchain_openai import AzureChatOpenAI
8-
from langchain_openai import AzureOpenAIEmbeddings
97
from scrapegraphai.graphs import JSONScraperGraph
108
from scrapegraphai.utils import prettify_exec_info
119

12-
13-
# required environment variable in .env
14-
# AZURE_OPENAI_ENDPOINT
15-
# AZURE_OPENAI_CHAT_DEPLOYMENT_NAME
16-
# MODEL_NAME
17-
# AZURE_OPENAI_API_KEY
18-
# OPENAI_API_TYPE
19-
# AZURE_OPENAI_API_VERSION
20-
# AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME
2110
load_dotenv()
2211

23-
2412
FILE_NAME = "inputs/example.json"
2513
curr_dir = os.path.dirname(os.path.realpath(__file__))
2614
file_path = os.path.join(curr_dir, FILE_NAME)
@@ -32,23 +20,13 @@
3220
# Initialize the model instances
3321
# ************************************************
3422

35-
llm_model_instance = AzureChatOpenAI(
36-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
37-
azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
38-
)
39-
40-
embedder_model_instance = AzureOpenAIEmbeddings(
41-
azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
42-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
43-
)
44-
45-
# ************************************************
46-
# Create the JSONScraperGraph instance and run it
47-
# ************************************************
48-
4923
graph_config = {
50-
"llm": {"model_instance": llm_model_instance},
51-
"embeddings": {"model_instance": embedder_model_instance}
24+
"llm": {
25+
"api_key": os.environ["AZURE_OPENAI_KEY"],
26+
"model": "azure/gpt-3.5-turbo",
27+
},
28+
"verbose": True,
29+
"headless": False
5230
}
5331

5432
smart_scraper_graph = JSONScraperGraph(

‎examples/azure/json_scraper_multi_azure.py

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,24 +2,22 @@
22
Module for showing how JSONScraperMultiGraph multi works
33
"""
44
import os
5+
from dotenv import load_dotenv
56
import json
6-
from langchain_openai import AzureChatOpenAI
7-
from langchain_openai import AzureOpenAIEmbeddings
87
from scrapegraphai.graphs import JSONScraperMultiGraph
98

10-
llm_model_instance = AzureChatOpenAI(
11-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
12-
azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
13-
)
9+
load_dotenv()
10+
1411

15-
embedder_model_instance = AzureOpenAIEmbeddings(
16-
azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
17-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
18-
)
1912
graph_config = {
20-
"llm": {"model_instance": llm_model_instance},
21-
"embeddings": {"model_instance": embedder_model_instance}
13+
"llm": {
14+
"api_key": os.environ["AZURE_OPENAI_KEY"],
15+
"model": "azure/gpt-3.5-turbo",
16+
},
17+
"verbose": True,
18+
"headless": False
2219
}
20+
2321
FILE_NAME = "inputs/example.json"
2422
curr_dir = os.path.dirname(os.path.realpath(__file__))
2523
file_path = os.path.join(curr_dir, FILE_NAME)

‎examples/azure/pdf_scraper_azure.py

Lines changed: 6 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,26 +1,19 @@
11
import os, json
22
from dotenv import load_dotenv
3-
from langchain_openai import AzureChatOpenAI
4-
from langchain_openai import AzureOpenAIEmbeddings
53
from scrapegraphai.graphs import PDFScraperGraph
64

75
load_dotenv()
86

97
# ************************************************
108
# Define the configuration for the graph
119
# ************************************************
12-
llm_model_instance = AzureChatOpenAI(
13-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
14-
azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
15-
)
16-
17-
embedder_model_instance = AzureOpenAIEmbeddings(
18-
azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
19-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
20-
)
2110
graph_config = {
22-
"llm": {"model_instance": llm_model_instance},
23-
"embeddings": {"model_instance": embedder_model_instance}
11+
"llm": {
12+
"api_key": os.environ["AZURE_OPENAI_KEY"],
13+
"model": "azure/gpt-3.5-turbo",
14+
},
15+
"verbose": True,
16+
"headless": False
2417
}
2518

2619
source = """

‎examples/azure/scrape_plain_text_azure.py

Lines changed: 6 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,6 @@
55
import os
66
from dotenv import load_dotenv
77
from scrapegraphai.graphs import SmartScraperGraph
8-
from langchain_openai import AzureChatOpenAI
9-
from langchain_openai import AzureOpenAIEmbeddings
108
from scrapegraphai.utils import prettify_exec_info
119

1210
load_dotenv()
@@ -27,23 +25,13 @@
2725
# Define the configuration for the graph
2826
# ************************************************
2927

30-
llm_model_instance = AzureChatOpenAI(
31-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
32-
azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
33-
)
34-
35-
embedder_model_instance = AzureOpenAIEmbeddings(
36-
azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
37-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
38-
)
39-
40-
# ************************************************
41-
# Create the JSONScraperGraph instance and run it
42-
# ************************************************
43-
4428
graph_config = {
45-
"llm": {"model_instance": llm_model_instance},
46-
"embeddings": {"model_instance": embedder_model_instance}
29+
"llm": {
30+
"api_key": os.environ["AZURE_OPENAI_KEY"],
31+
"model": "azure/gpt-3.5-turbo",
32+
},
33+
"verbose": True,
34+
"headless": False
4735
}
4836

4937
# ************************************************

‎examples/azure/script_generator_azure.py

Lines changed: 6 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,28 +5,20 @@
55
import os
66
from dotenv import load_dotenv
77
from scrapegraphai.graphs import ScriptCreatorGraph
8-
from langchain_openai import AzureChatOpenAI
9-
from langchain_openai import AzureOpenAIEmbeddings
108
from scrapegraphai.utils import prettify_exec_info
119

1210
load_dotenv()
1311

1412
# ************************************************
1513
# Define the configuration for the graph
1614
# ************************************************
17-
llm_model_instance = AzureChatOpenAI(
18-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
19-
azure_deployment=os.environ["AZURE_OPENAI_CHAT_DEPLOYMENT_NAME"]
20-
)
21-
22-
embedder_model_instance = AzureOpenAIEmbeddings(
23-
azure_deployment=os.environ["AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME"],
24-
openai_api_version=os.environ["AZURE_OPENAI_API_VERSION"],
25-
)
2615
graph_config = {
27-
"llm": {"model_instance": llm_model_instance},
28-
"embeddings": {"model_instance": embedder_model_instance},
29-
"library": "beautifulsoup"
16+
"llm": {
17+
"api_key": os.environ["AZURE_OPENAI_KEY"],
18+
"model": "azure/gpt-3.5-turbo",
19+
},
20+
"verbose": True,
21+
"headless": False
3022
}
3123

3224
# ************************************************

0 commit comments

Comments
 (0)