Skip to content

Commit 88a904b

Browse files
nsarrazinnsarrazin
authored
feat(models): Llama 3.1 support (#1355)
* feat(models): Add llama 3.1 405B and 70B * feat(tools): improve tools support for llama 3.1 models * feat(models): update mistral to latest * feat(models): use llama 3.1 8B for tasks * fix(tools): disable tools by default * feat(models): update model descriptions in huggingchat * fix(front): show branding for llama 3 * fix(config): fix encoding * feat(migrations): reset tools for everyone * feat(config): replace orgs with final org * feat(config): update website for llama 3.1
1 parent a1b0474 commit 88a904b

File tree

13 files changed

+174
-80
lines changed

13 files changed

+174
-80
lines changed

‎chart/env/prod.yaml

Lines changed: 74 additions & 42 deletions
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,68 @@ envVars:
3838
METRICS_ENABLED: "true"
3939
MODELS: >
4040
[
41+
{
42+
"name" : "meta-llama/Meta-Llama-3.1-70B-Instruct",
43+
"id": "meta-llama/Meta-Llama-3.1-70B-Instruct",
44+
"tokenizer": {"tokenizerUrl": "https://huggingface.co/nsarrazin/llama3.1-tokenizer/resolve/main/tokenizer.json", "tokenizerConfigUrl": "https://huggingface.co/nsarrazin/llama3.1-tokenizer/raw/main/tokenizer_config.json"},
45+
"description": "Ideal for everyday use. A fast and extremely capable model matching closed source models' capabilities.",
46+
"modelUrl": "https://huggingface.co/meta-llama/Meta-Llama-3.1-70B-Instruct",
47+
"websiteUrl": "https://llama.meta.com/",
48+
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/meta-logo.png",
49+
"tools": true,
50+
"preprompt" : "You are a helpful assistant with tool calling capabilities. The user has access to the tool's outputs that you as a model cannot see. This could include text, images and more.",
51+
"parameters": {
52+
"temperature": 0.1,
53+
"stop": ["<|endoftext|>", "<|eot_id|>"],
54+
"max_new_tokens": 1024,
55+
"truncate": 7167
56+
},
57+
"promptExamples": [
58+
{
59+
"title": "Write an email from bullet list",
60+
"prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
61+
}, {
62+
"title": "Code a snake game",
63+
"prompt": "Code a basic snake game in python, give explanations for each step."
64+
}, {
65+
"title": "Assist in a task",
66+
"prompt": "How do I make a delicious lemon cheesecake?"
67+
}
68+
]
69+
},
70+
{
71+
"name" : "meta-llama/Meta-Llama-3.1-405B-Instruct-FP8",
72+
"id": "meta-llama/Meta-Llama-3.1-405B-Instruct-FP8",
73+
"tokenizer": {"tokenizerUrl": "https://huggingface.co/nsarrazin/llama3.1-tokenizer/resolve/main/tokenizer.json", "tokenizerConfigUrl": "https://huggingface.co/nsarrazin/llama3.1-tokenizer/raw/main/tokenizer_config.json"},
74+
"description": "The most intelligent open-source model, showing exceptional capabilities for complex tasks.",
75+
"modelUrl": "https://huggingface.co/meta-llama/Meta-Llama-3.1-405B-Instruct-FP8",
76+
"websiteUrl": "https://llama.meta.com/",
77+
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/meta-logo.png",
78+
"tools": true,
79+
"preprompt" : "You are a helpful assistant with tool calling capabilities. The user has access to the tool's outputs that you as a model cannot see. This could include text, images and more.",
80+
"parameters": {
81+
"temperature": 0.1,
82+
"stop": ["<|endoftext|>", "<|eot_id|>"],
83+
"max_new_tokens": 2048,
84+
"truncate": 14337
85+
},
86+
"promptExamples": [
87+
{
88+
"title": "Write an email from bullet list",
89+
"prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
90+
}, {
91+
"title": "Code a snake game",
92+
"prompt": "Code a basic snake game in python, give explanations for each step."
93+
}, {
94+
"title": "Assist in a task",
95+
"prompt": "How do I make a delicious lemon cheesecake?"
96+
}
97+
]
98+
},
4199
{
42100
"name" : "CohereForAI/c4ai-command-r-plus",
43101
"tokenizer": {"tokenizerUrl": "https://huggingface.co/nsarrazin/c4ai-command-r-v01-tokenizer/resolve/main/tokenizer.json", "tokenizerConfigUrl": "https://huggingface.co/nsarrazin/c4ai-command-r-v01-tokenizer/raw/main/tokenizer_config.json"},
44-
"description": "Command R+ is Cohere's latest LLM and is the first open weight model to beat GPT4 in the Chatbot Arena!",
102+
"description": "Cohere's largest language model, optimized for conversational interaction and tool use.",
45103
"modelUrl": "https://huggingface.co/CohereForAI/c4ai-command-r-plus",
46104
"websiteUrl": "https://docs.cohere.com/docs/command-r-plus",
47105
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/cohere-logo.png",
@@ -65,36 +123,9 @@ envVars:
65123
}
66124
]
67125
},
68-
{
69-
"name" : "meta-llama/Meta-Llama-3-70B-Instruct",
70-
"description": "Meta Llama 3 delivers top performance on various benchmarks and introduces new features like better reasoning.",
71-
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/meta-logo.png",
72-
"modelUrl": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct",
73-
"websiteUrl": "https://llama.meta.com/llama3/",
74-
"tokenizer" : "philschmid/meta-llama-3-tokenizer",
75-
"promptExamples" : [
76-
{
77-
"title": "Write an email from bullet list",
78-
"prompt": "As a restaurant owner, write a professional email to the supplier to get these products every week: \n\n- Wine (x10)\n- Eggs (x24)\n- Bread (x12)"
79-
}, {
80-
"title": "Code a snake game",
81-
"prompt": "Code a basic snake game in python, give explanations for each step."
82-
}, {
83-
"title": "Assist in a task",
84-
"prompt": "How do I make a delicious lemon cheesecake?"
85-
}
86-
],
87-
"parameters": {
88-
"stop": ["<|eot_id|>"],
89-
"truncate": 6144,
90-
"max_new_tokens": 2047,
91-
"temperature": 0.6,
92-
"top_p" : 0.9
93-
}
94-
},
95126
{
96127
"name" : "mistralai/Mixtral-8x7B-Instruct-v0.1",
97-
"description" : "The latest MoE model from Mistral AI! 8x7B and outperforms Llama 2 70B in most benchmarks.",
128+
"description" : "A high-quality sparse mixture of experts model with open weights.",
98129
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/mistral-logo.png",
99130
"websiteUrl" : "https://mistral.ai/news/mixtral-of-experts/",
100131
"modelUrl": "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1",
@@ -125,7 +156,7 @@ envVars:
125156
},
126157
{
127158
"name" : "NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
128-
"description" : "Nous Hermes 2 Mixtral 8x7B DPO is the new flagship Nous Research model trained over the Mixtral 8x7B MoE LLM.",
159+
"description" : "Nous Hermes' strong flagship model trained on the Mixtral 8x7B.",
129160
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/nous-logo.png",
130161
"websiteUrl" : "https://nousresearch.com/",
131162
"modelUrl": "https://huggingface.co/NousResearch/Nous-Hermes-2-Mixtral-8x7B-DPO",
@@ -156,7 +187,7 @@ envVars:
156187
{
157188
"name": "01-ai/Yi-1.5-34B-Chat",
158189
"tokenizer": "01-ai/Yi-1.5-34B-Chat",
159-
"description" : "Yi-1.5 is an upgraded version of Yi. It is continuously pre-trained on Yi with a high-quality corpus of 500B tokens and fine-tuned on 3M diverse fine-tuning samples.",
190+
"description" : "Strong performance in reasoning while maintaining excellent capabilities in language understanding.",
160191
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/01-ai-logo.png",
161192
"modelUrl": "https://huggingface.co/01-ai/Yi-1.5-34B-Chat",
162193
"websiteUrl": "https://www.01.ai",
@@ -182,13 +213,13 @@ envVars:
182213
]
183214
},
184215
{
185-
"name": "mistralai/Mistral-7B-Instruct-v0.2",
186-
"displayName": "mistralai/Mistral-7B-Instruct-v0.2",
187-
"description": "Mistral 7B is a new Apache 2.0 model, released by Mistral AI that outperforms Llama2 13B in benchmarks.",
216+
"name": "mistralai/Mistral-7B-Instruct-v0.3",
217+
"displayName": "mistralai/Mistral-7B-Instruct-v0.3",
218+
"description": "A small model with good capabilities in language understanding and commonsense reasoning.",
188219
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/mistral-logo.png",
189220
"websiteUrl": "https://mistral.ai/news/announcing-mistral-7b/",
190-
"modelUrl": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.2",
191-
"tokenizer": "mistralai/Mistral-7B-Instruct-v0.2",
221+
"modelUrl": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.3",
222+
"tokenizer": "mistralai/Mistral-7B-Instruct-v0.3",
192223
"preprompt": "",
193224
"chatPromptTemplate" : "<s>{{#each messages}}{{#ifUser}}[INST] {{#if @first}}{{#if @root.preprompt}}{{@root.preprompt}}\n{{/if}}{{/if}}{{content}} [/INST]{{/ifUser}}{{#ifAssistant}}{{content}}</s>{{/ifAssistant}}{{/each}}",
194225
"parameters": {
@@ -216,7 +247,7 @@ envVars:
216247
{
217248
"name": "microsoft/Phi-3-mini-4k-instruct",
218249
"tokenizer": "microsoft/Phi-3-mini-4k-instruct",
219-
"description" : "Phi-3 Mini-4K-Instruct is a 3.8B parameters, lightweight, state-of-the-art open model built upon datasets used for Phi-2.",
250+
"description" : "One of the best small models (3.8B parameters), super fast for simple tasks.",
220251
"logoUrl": "https://huggingface.co/datasets/huggingchat/models-logo/resolve/main/microsoft-logo.png",
221252
"modelUrl": "https://huggingface.co/microsoft/Phi-3-mini-4k-instruct",
222253
"websiteUrl": "https://azure.microsoft.com/en-us/blog/introducing-phi-3-redefining-whats-possible-with-slms/",
@@ -242,11 +273,11 @@ envVars:
242273
]
243274
},
244275
{
245-
"name": "meta-llama/Meta-Llama-3-8B-Instruct",
246-
"tokenizer" : "philschmid/meta-llama-3-tokenizer",
276+
"name" : "llhf/Meta-Llama-3.1-8B-Instruct",
277+
"tokenizer": {"tokenizerUrl": "https://huggingface.co/nsarrazin/llama3.1-tokenizer/resolve/main/tokenizer.json", "tokenizerConfigUrl": "https://huggingface.co/nsarrazin/llama3.1-tokenizer/raw/main/tokenizer_config.json"},
247278
"parameters": {
248279
"temperature": 0.1,
249-
"stop": ["<|eot_id|>"],
280+
"stop": ["<|endoftext|>", "<|eot_id|>"],
250281
},
251282
"unlisted": true
252283
}
@@ -265,7 +296,8 @@ envVars:
265296
{ "name": "google/gemma-7b-it" },
266297
{ "name": "meta-llama/Llama-2-70b-chat-hf" },
267298
{ "name": "codellama/CodeLlama-70b-Instruct-hf" },
268-
{ "name": "openchat/openchat-3.5-0106" }
299+
{ "name": "openchat/openchat-3.5-0106" },
300+
{ "name": "meta-llama/Meta-Llama-3-70B-Instruct"}
269301
]
270302
PUBLIC_ORIGIN: "https://huggingface.co"
271303
PUBLIC_SHARE_PREFIX: "https://hf.co/chat"
@@ -280,7 +312,7 @@ envVars:
280312
PUBLIC_PLAUSIBLE_SCRIPT_URL: "/js/script.js"
281313
PUBLIC_APPLE_APP_ID: "6476778843"
282314
REQUIRE_FEATURED_ASSISTANTS: "true"
283-
TASK_MODEL: "meta-llama/Meta-Llama-3-8B-Instruct"
315+
TASK_MODEL: "llhf/Meta-Llama-3.1-8B-Instruct"
284316
TEXT_EMBEDDING_MODELS: >
285317
[{
286318
"name": "bge-base-en-v1-5-sxa",

‎src/lib/components/ModelCardMetadata.svelte

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,9 +42,9 @@
4242
class="ml-auto flex items-center hover:underline"
4343
rel="noreferrer"
4444
>
45-
{#if model.name === "meta-llama/Meta-Llama-3-70B-Instruct"}
45+
{#if model.name.startsWith("meta-llama/Meta-Llama")}
4646
<BIMeta class="mr-1.5 shrink-0 text-xs text-gray-400" />
47-
Built with Meta Llama 3
47+
Built with Llama
4848
{:else}
4949
<CarbonEarth class="mr-1.5 shrink-0 text-xs text-gray-400" />
5050
Website

‎src/lib/migrations/routines/07-reset-tools-in-settings.ts

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
import type { Migration } from ".";
2+
import { collections } from "$lib/server/database";
3+
import { ObjectId } from "mongodb";
4+
5+
const resetTools: Migration = {
6+
_id: new ObjectId("000000000007"),
7+
name: "Reset tools to empty",
8+
up: async () => {
9+
const { settings } = collections;
10+
11+
await settings.updateMany({}, { $set: { tools: {} } });
12+
13+
return true;
14+
},
15+
runEveryTime: false,
16+
runForHuggingChat: "only",
17+
};
18+
19+
export default resetTools;

‎src/lib/migrations/routines/index.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import addToolsToSettings from "./03-add-tools-in-settings";
77
import updateMessageUpdates from "./04-update-message-updates";
88
import updateMessageFiles from "./05-update-message-files";
99
import trimMessageUpdates from "./06-trim-message-updates";
10+
import resetTools from "./07-reset-tools-in-settings";
1011

1112
export interface Migration {
1213
_id: ObjectId;
@@ -25,4 +26,5 @@ export const migrations: Migration[] = [
2526
updateMessageUpdates,
2627
updateMessageFiles,
2728
trimMessageUpdates,
29+
resetTools,
2830
];

‎src/lib/server/models.ts

Lines changed: 65 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -96,7 +96,10 @@ async function getChatPromptRender(
9696

9797
const renderTemplate = ({ messages, preprompt, tools, toolResults }: ChatTemplateInput) => {
9898
let formattedMessages: { role: string; content: string }[] = messages.map((message) => ({
99-
content: message.content,
99+
content:
100+
message.files?.length && !tools?.length
101+
? message.content + `\n This message has ${message.files.length} files attached`
102+
: message.content,
100103
role: message.from,
101104
}));
102105

@@ -113,32 +116,69 @@ async function getChatPromptRender(
113116
if (toolResults?.length) {
114117
// todo: should update the command r+ tokenizer to support system messages at any location
115118
// or use the `rag` mode without the citations
116-
formattedMessages = [
117-
{
118-
role: "system",
119-
content:
120-
"\n\n<results>\n" +
121-
toolResults
122-
.flatMap((result, idx) => {
123-
if (result.status === ToolResultStatus.Error) {
119+
const id = m.id ?? m.name;
120+
121+
if (id.startsWith("CohereForAI")) {
122+
formattedMessages = [
123+
{
124+
role: "system",
125+
content:
126+
"\n\n<results>\n" +
127+
toolResults
128+
.flatMap((result, idx) => {
129+
if (result.status === ToolResultStatus.Error) {
130+
return (
131+
`Document: ${idx}\n` + `Tool "${result.call.name}" error\n` + result.message
132+
);
133+
}
124134
return (
125-
`Document: ${idx}\n` + `Tool "${result.call.name}" error\n` + result.message
135+
`Document: ${idx}\n` +
136+
result.outputs
137+
.flatMap((output) =>
138+
Object.entries(output).map(([title, text]) => `${title}\n${text}`)
139+
)
140+
.join("\n")
126141
);
127-
}
128-
return (
129-
`Document: ${idx}\n` +
130-
result.outputs
131-
.flatMap((output) =>
132-
Object.entries(output).map(([title, text]) => `${title}\n${text}`)
133-
)
134-
.join("\n")
135-
);
136-
})
137-
.join("\n\n") +
138-
"\n</results>",
139-
},
140-
...formattedMessages,
141-
];
142+
})
143+
.join("\n\n") +
144+
"\n</results>",
145+
},
146+
...formattedMessages,
147+
];
148+
} else if (id.startsWith("meta-llama")) {
149+
const results = toolResults.flatMap((result) => {
150+
if (result.status === ToolResultStatus.Error) {
151+
return [
152+
{
153+
tool_call_id: result.call.name,
154+
output: "Error: " + result.message,
155+
},
156+
];
157+
} else {
158+
logger.info(result.outputs);
159+
return result.outputs.map((output) => ({
160+
tool_call_id: result.call.name,
161+
output: JSON.stringify(output),
162+
}));
163+
}
164+
});
165+
166+
formattedMessages = [
167+
...formattedMessages,
168+
{
169+
role: "python",
170+
content: JSON.stringify(results),
171+
},
172+
];
173+
} else {
174+
formattedMessages = [
175+
...formattedMessages,
176+
{
177+
role: "system",
178+
content: JSON.stringify(toolResults),
179+
},
180+
];
181+
}
142182
tools = [];
143183
}
144184

‎src/lib/server/textGeneration/tools.ts

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -143,19 +143,26 @@ export async function* runTools(
143143
// look for a code blocks of ```json and parse them
144144
// if they're valid json, add them to the calls array
145145
if (output.generated_text) {
146+
if (!output.generated_text.endsWith("```")) {
147+
output.generated_text = output.generated_text + "```";
148+
}
146149
const codeBlocks = Array.from(output.generated_text.matchAll(/```json\n(.*?)```/gs))
147150
.map(([, block]) => block)
148151
// remove trailing comma
149152
.map((block) => block.trim().replace(/,$/, ""));
150153
if (codeBlocks.length === 0) continue;
151-
152154
// grab only the capture group from the regex match
153155
for (const block of codeBlocks) {
156+
// make it an array if it's not already
157+
let call = JSON5.parse(block);
158+
if (!Array.isArray(call)) {
159+
call = [call];
160+
}
161+
154162
try {
155-
calls.push(
156-
...JSON5.parse(block).filter(isExternalToolCall).map(externalToToolCall).filter(Boolean)
157-
);
163+
calls.push(...call.filter(isExternalToolCall).map(externalToToolCall).filter(Boolean));
158164
} catch (e) {
165+
logger.error(e, "Error while parsing tool calls, please retry");
159166
// error parsing the calls
160167
yield {
161168
type: MessageUpdateType.Status,

‎src/lib/server/tools/calculator.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,6 @@ const calculator: BackendTool = {
66
displayName: "Calculator",
77
description:
88
"A simple calculator, takes a string containing a mathematical expression and returns the answer. Only supports +, -, *, ** (power) and /, as well as parenthesis ().",
9-
isOnByDefault: true,
109
parameterDefinitions: {
1110
equation: {
1211
description:

‎src/lib/server/tools/documentParser.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@ const documentParser: BackendTool = {
99
name: "document_parser",
1010
displayName: "Document Parser",
1111
description: "Use this tool to parse any document and get its content in markdown format.",
12-
isOnByDefault: true,
1312
mimeTypes: ["application/*", "text/*"],
1413
parameterDefinitions: {
1514
fileMessageIndex: {

‎src/lib/server/tools/images/editing.ts

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@ const imageEditing: BackendTool = {
1717
name: "image_editing",
1818
displayName: "Image Editing",
1919
description: "Use this tool to edit an image from a prompt.",
20-
isOnByDefault: true,
2120
mimeTypes: ["image/*"],
2221
parameterDefinitions: {
2322
prompt: {

0 commit comments

Comments
 (0)