add new models and import

Author: VinciGit00

scrapegraphai/helpers/mappings.py

@@ -5,6 +5,7 @@
     "llama2": "isenbek/lama-2-7b-chat-hf-local-1",
     "llama3": "meta-llama/Meta-Llama-3-8B",
     "llama3:70b": "meta-llama/Meta-Llama-3-70B",
+    "llama3.1":"meta-llama/Meta-Llama-3.1-8B",
     "llama3.1:70b":"meta-llama/Meta-Llama-3.1-70B",
     "mistral": "mistralai/Mistral-Nemo-Instruct-2407"
-}
+}

scrapegraphai/nodes/parse_node.py

@@ -15,6 +15,7 @@
 from ..helpers import models_tokens
 from ..utils.tokenizer_openai import num_tokens_openai
 from .base_node import BaseNode
+from ..helpers.mappings import translation_hf
 
 class ParseNode(BaseNode):
     """
@@ -102,7 +103,7 @@ def execute(self, state: dict) -> dict:
             elif isinstance(self.llm_model, ChatMistralAI):
                 print("mistral")
             elif isinstance(self.llm_model, ChatOllama):
-                tokenizer = AutoTokenizer.from_pretrained("meta-llama/Meta-Llama-3-8B")
+                tokenizer = AutoTokenizer.from_pretrained(translation_hf[self.llm_model.split("/")[-1]])
                 tokens = tokenizer.tokenize(docs_transformed.page_conten)
                 num_tokens = len(tokens)
             #google genai