Merge pull request #583 from ScrapeGraphAI/anthropic-refactoring

VinciGit00 · web-flow · commit 4dea97299841 · 2024-08-25T10:54:11.000+02:00
Anthropic refactoring
diff --git a/examples/anthropic/csv_scraper_anthropic.py b/examples/anthropic/csv_scraper_anthropic.py
@@ -32,7 +32,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/csv_scraper_graph_multi_anthropic.py b/examples/anthropic/csv_scraper_graph_multi_anthropic.py
@@ -26,7 +26,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000},
 }
 
diff --git a/examples/anthropic/custom_graph_anthropic.py b/examples/anthropic/custom_graph_anthropic.py
@@ -18,7 +18,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/json_scraper_anthropic.py b/examples/anthropic/json_scraper_anthropic.py
@@ -26,7 +26,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/json_scraper_multi_anthropic.py b/examples/anthropic/json_scraper_multi_anthropic.py
@@ -11,7 +11,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/pdf_scraper_graph_anthropic.py b/examples/anthropic/pdf_scraper_graph_anthropic.py
@@ -14,7 +14,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/pdf_scraper_multi_anthropic.py b/examples/anthropic/pdf_scraper_multi_anthropic.py
@@ -11,7 +11,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/scrape_plain_text_anthropic.py b/examples/anthropic/scrape_plain_text_anthropic.py
@@ -28,7 +28,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/script_generator_anthropic.py b/examples/anthropic/script_generator_anthropic.py
@@ -16,7 +16,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/script_multi_generator_anthropic.py b/examples/anthropic/script_multi_generator_anthropic.py
@@ -16,7 +16,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
         "library": "beautifulsoup"
diff --git a/examples/anthropic/search_graph_anthropic.py b/examples/anthropic/search_graph_anthropic.py
@@ -15,7 +15,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/search_graph_schema_anthropic.py b/examples/anthropic/search_graph_schema_anthropic.py
@@ -27,8 +27,9 @@ class Dishes(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
-        "max_tokens": 4000},
+        "model": "anthropic/claude-3-haiku-20240307",
+        "max_tokens": 4000
+        },
 }
 
 # ************************************************
diff --git a/examples/anthropic/search_link_graph_anthropic.py b/examples/anthropic/search_link_graph_anthropic.py
@@ -29,8 +29,11 @@
 # ************************************************
 
 graph_config = {
-    "llm": {"model_instance": llm_model_instance},
-    "embeddings": {"model_instance": embedder_model_instance}
+    "llm": {
+        "api_key": os.getenv("ANTHROPIC_API_KEY"),
+        "model": "anthropic/claude-3-haiku-20240307",
+        "max_tokens": 4000
+        },
 }
 
 # ************************************************
diff --git a/examples/anthropic/smart_scraper_anthropic.py b/examples/anthropic/smart_scraper_anthropic.py
@@ -19,7 +19,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/smart_scraper_multi_anthropic.py b/examples/anthropic/smart_scraper_multi_anthropic.py
@@ -17,7 +17,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/smart_scraper_schema_anthropic.py b/examples/anthropic/smart_scraper_schema_anthropic.py
@@ -33,7 +33,7 @@ class Projects(BaseModel):
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000},
 }
 
diff --git a/examples/anthropic/xml_scraper_anthropic.py b/examples/anthropic/xml_scraper_anthropic.py
@@ -26,7 +26,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000
         },
 }
diff --git a/examples/anthropic/xml_scraper_graph_multi_anthropic.py b/examples/anthropic/xml_scraper_graph_multi_anthropic.py
@@ -26,7 +26,7 @@
 graph_config = {
     "llm": {
         "api_key": os.getenv("ANTHROPIC_API_KEY"),
-        "model": "claude-3-haiku-20240307",
+        "model": "anthropic/claude-3-haiku-20240307",
         "max_tokens": 4000},
 }
 
diff --git a/scrapegraphai/graphs/abstract_graph.py b/scrapegraphai/graphs/abstract_graph.py
@@ -142,7 +142,7 @@ def handle_model(model_name, provider, token_key, default_token=8192):
         known_models = {"chatgpt","gpt","openai", "azure_openai", "google_genai",
                         "ollama", "oneapi", "nvidia", "groq", "google_vertexai",
                         "bedrock", "mistralai", "hugging_face", "deepseek", "ernie", 
-                        "fireworks", "claude-3-"}
+                        "fireworks", "anthropic"}
 
         if llm_params["model"].split("/")[0] not in known_models and llm_params["model"].split("-")[0] not in known_models:
             raise ValueError(f"Model '{llm_params['model']}' is not supported")
@@ -172,8 +172,9 @@ def handle_model(model_name, provider, token_key, default_token=8192):
                 token_key = model_name if "model_tokens" not in llm_params else llm_params["model_tokens"]
                 return handle_model(model_name, "ollama", token_key)
 
-            elif "claude-3-" in llm_params["model"]:
-                return handle_model(llm_params["model"], "anthropic", "claude3")
+            elif "anthropic" in llm_params["model"]:
+                model_name = llm_params["model"].split("anthropic/")[-1]
+                return handle_model(model_name, "anthropic", model_name)
 
             elif llm_params["model"].startswith("mistral"):
                 model_name = llm_params["model"].split("/")[-1]

Original file line number	Diff line number	Diff line change
`@@ -32,7 +32,7 @@`
`32`	`32`	`graph_config = {`
`33`	`33`	`"llm": {`
`34`	`34`	`"api_key": os.getenv("ANTHROPIC_API_KEY"),`
`35`		`- "model": "claude-3-haiku-20240307",`
	`35`	`+ "model": "anthropic/claude-3-haiku-20240307",`
`36`	`36`	`"max_tokens": 4000`
`37`	`37`	`},`
`38`	`38`	`}`
Original file line number	Diff line number	Diff line change
`@@ -26,7 +26,7 @@`
`26`	`26`	`graph_config = {`
`27`	`27`	`"llm": {`
`28`	`28`	`"api_key": os.getenv("ANTHROPIC_API_KEY"),`
`29`		`- "model": "claude-3-haiku-20240307",`
	`29`	`+ "model": "anthropic/claude-3-haiku-20240307",`
`30`	`30`	`"max_tokens": 4000},`
`31`	`31`	`}`
`32`	`32`
Original file line number	Diff line number	Diff line change
`@@ -18,7 +18,7 @@`
`18`	`18`	`graph_config = {`
`19`	`19`	`"llm": {`
`20`	`20`	`"api_key": os.getenv("ANTHROPIC_API_KEY"),`
`21`		`- "model": "claude-3-haiku-20240307",`
	`21`	`+ "model": "anthropic/claude-3-haiku-20240307",`
`22`	`22`	`"max_tokens": 4000`
`23`	`23`	`},`
`24`	`24`	`}`
Original file line number	Diff line number	Diff line change
`@@ -11,7 +11,7 @@`
`11`	`11`	`graph_config = {`
`12`	`12`	`"llm": {`
`13`	`13`	`"api_key": os.getenv("ANTHROPIC_API_KEY"),`
`14`		`- "model": "claude-3-haiku-20240307",`
	`14`	`+ "model": "anthropic/claude-3-haiku-20240307",`
`15`	`15`	`"max_tokens": 4000`
`16`	`16`	`},`
`17`	`17`	`}`
Original file line number	Diff line number	Diff line change
`@@ -14,7 +14,7 @@`
`14`	`14`	`graph_config = {`
`15`	`15`	`"llm": {`
`16`	`16`	`"api_key": os.getenv("ANTHROPIC_API_KEY"),`
`17`		`- "model": "claude-3-haiku-20240307",`
	`17`	`+ "model": "anthropic/claude-3-haiku-20240307",`
`18`	`18`	`"max_tokens": 4000`
`19`	`19`	`},`
`20`	`20`	`}`
Original file line number	Diff line number	Diff line change
`@@ -28,7 +28,7 @@`
`28`	`28`	`graph_config = {`
`29`	`29`	`"llm": {`
`30`	`30`	`"api_key": os.getenv("ANTHROPIC_API_KEY"),`
`31`		`- "model": "claude-3-haiku-20240307",`
	`31`	`+ "model": "anthropic/claude-3-haiku-20240307",`
`32`	`32`	`"max_tokens": 4000`
`33`	`33`	`},`
`34`	`34`	`}`
Original file line number	Diff line number	Diff line change
`@@ -16,7 +16,7 @@`
`16`	`16`	`graph_config = {`
`17`	`17`	`"llm": {`
`18`	`18`	`"api_key": os.getenv("ANTHROPIC_API_KEY"),`
`19`		`- "model": "claude-3-haiku-20240307",`
	`19`	`+ "model": "anthropic/claude-3-haiku-20240307",`
`20`	`20`	`"max_tokens": 4000`
`21`	`21`	`},`
`22`	`22`	`}`