From 4e23a832692dd4675e4b7f97bc392fd07eeb8070 Mon Sep 17 00:00:00 2001 From: Julien Denize Date: Thu, 17 Jul 2025 15:29:55 +0200 Subject: [PATCH 1/5] Add vLLM library --- .../tasks/src/model-libraries-snippets.ts | 22 +++++++++++++++++++ packages/tasks/src/model-libraries.ts | 8 +++++++ 2 files changed, 30 insertions(+) diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index 9782c3bfea..33c3e93d82 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -1893,4 +1893,26 @@ audio = model.autoencoder.decode(codes)[0].cpu() torchaudio.save("sample.wav", audio, model.autoencoder.sampling_rate) `, ]; + +export const vllm = (model: ModelData): string[] => [ + `pip install --upgrade vllm + +# If serving Mistral AI models, make sure to update mistral-common: +# pip install mistral-common --upgrade + +# serve the model with an OpenAI-compatible API +vllm serve ${model.id} + +# for Mistral AI models, use: +# vllm serve ${model.id} --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice + +# query the model with curl in a separate terminal +curl http://localhost:8000/v1/chat/completions \ + -H "Content-Type: application/json" \ + -d '{ + "model": "${model.id}", + "messages": [{"role": "user", "content": "What is the capital of France?"}] + }'`, +]; + //#endregion diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index 42002bcf92..0b578f48bc 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -1196,6 +1196,14 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { countDownloads: `path:"model_vae_fp16.pt"`, snippets: snippets.threedtopia_xl, }, + vllm: { + prettyLabel: "vLLM", + repoName: "vllm", + repoUrl: "https://github.com/vllm-project/vllm", + docsUrl: "https://docs.vllm.ai/en/stable/", + snippets: snippets.vllm, + countDownloads: `path:"config.json" OR path:"params.json"`, + }, } satisfies Record; export type ModelLibraryKey = keyof typeof MODEL_LIBRARIES_UI_ELEMENTS; From 4cd3f273cbe09505dfcbedc3c175d38b9f900a82 Mon Sep 17 00:00:00 2001 From: Julien Denize Date: Fri, 18 Jul 2025 14:05:03 +0200 Subject: [PATCH 2/5] Add mistral-common library --- packages/tasks/src/local-apps.ts | 22 +++++++++++++++++-- .../tasks/src/model-libraries-snippets.ts | 18 +++++++-------- packages/tasks/src/model-libraries.ts | 16 +++++++------- 3 files changed, 36 insertions(+), 20 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index 53f6880282..f2eadfb60c 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -207,10 +207,28 @@ curl -X POST "http://localhost:8000/v1/completions" \\ "temperature": 0.5 }'`; const runCommand = model.tags.includes("conversational") ? runCommandInstruct : runCommandNonInstruct; + + let setup; + let dockerCommand; + + if (model.tags.includes("mistral-common")) { + setup = [ + "# Install vLLM from pip:", + "pip install vllm", + "# Make sure you have the latest version of mistral-common installed:", + "pip install --upgrade mistral-common" + ].join(""); + dockerCommand = `# Load and run the model:\ndocker exec -it my_vllm_container bash -c "vllm serve ${model.id} --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice"`; + } + else { + setup = ["# Install vLLM from pip:", "pip install vllm"].join(""); + dockerCommand = `# Load and run the model:\ndocker exec -it my_vllm_container bash -c "vllm serve ${model.id}"`; + } + return [ { title: "Install from pip", - setup: ["# Install vLLM from pip:", "pip install vllm"].join("\n"), + setup: setup, content: [`# Load and run the model:\nvllm serve "${model.id}"`, runCommand], }, { @@ -227,7 +245,7 @@ curl -X POST "http://localhost:8000/v1/completions" \\ ` --model ${model.id}`, ].join("\n"), content: [ - `# Load and run the model:\ndocker exec -it my_vllm_container bash -c "vllm serve ${model.id}"`, + dockerCommand, runCommand, ], }, diff --git a/packages/tasks/src/model-libraries-snippets.ts b/packages/tasks/src/model-libraries-snippets.ts index 33c3e93d82..f81844d886 100644 --- a/packages/tasks/src/model-libraries-snippets.ts +++ b/packages/tasks/src/model-libraries-snippets.ts @@ -1894,19 +1894,17 @@ torchaudio.save("sample.wav", audio, model.autoencoder.sampling_rate) `, ]; -export const vllm = (model: ModelData): string[] => [ - `pip install --upgrade vllm - -# If serving Mistral AI models, make sure to update mistral-common: -# pip install mistral-common --upgrade +export const mistral_common = (model: ModelData): string[] => [ + `# We recommend to use vLLM to serve Mistral AI models. +pip install vllm -# serve the model with an OpenAI-compatible API -vllm serve ${model.id} +# Make sure to have installed the latest version of mistral-common. +pip install --upgrade mistral-common[image,audio] -# for Mistral AI models, use: -# vllm serve ${model.id} --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice +# Serve the model with an OpenAI-compatible API. +vllm serve ${model.id} --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice -# query the model with curl in a separate terminal +# Query the model with curl in a separate terminal. curl http://localhost:8000/v1/chat/completions \ -H "Content-Type: application/json" \ -d '{ diff --git a/packages/tasks/src/model-libraries.ts b/packages/tasks/src/model-libraries.ts index 0b578f48bc..e68baf4d44 100644 --- a/packages/tasks/src/model-libraries.ts +++ b/packages/tasks/src/model-libraries.ts @@ -630,6 +630,14 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { filter: false, countDownloads: `path_extension:"ckpt"`, }, + mistral_common: { + prettyLabel: "mistral-common", + repoName: "mistral-common", + repoUrl: "https://github.com/mistralai/mistral-common", + docsUrl: "https://mistralai.github.io/mistral-common/", + snippets: snippets.mistral_common, + countDownloads: `path:"config.json" OR path:"params.json"`, + }, mitie: { prettyLabel: "MITIE", repoName: "MITIE", @@ -1196,14 +1204,6 @@ export const MODEL_LIBRARIES_UI_ELEMENTS = { countDownloads: `path:"model_vae_fp16.pt"`, snippets: snippets.threedtopia_xl, }, - vllm: { - prettyLabel: "vLLM", - repoName: "vllm", - repoUrl: "https://github.com/vllm-project/vllm", - docsUrl: "https://docs.vllm.ai/en/stable/", - snippets: snippets.vllm, - countDownloads: `path:"config.json" OR path:"params.json"`, - }, } satisfies Record; export type ModelLibraryKey = keyof typeof MODEL_LIBRARIES_UI_ELEMENTS; From df8ee13e414afaab25e4249c6f7aa775bbca585c Mon Sep 17 00:00:00 2001 From: Julien Denize <40604584+juliendenize@users.noreply.github.com> Date: Wed, 23 Jul 2025 13:57:05 +0200 Subject: [PATCH 3/5] Update packages/tasks/src/local-apps.ts Co-authored-by: Lucain --- packages/tasks/src/local-apps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index f2eadfb60c..19ce1c3379 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -217,7 +217,7 @@ curl -X POST "http://localhost:8000/v1/completions" \\ "pip install vllm", "# Make sure you have the latest version of mistral-common installed:", "pip install --upgrade mistral-common" - ].join(""); + ].join("\n"); dockerCommand = `# Load and run the model:\ndocker exec -it my_vllm_container bash -c "vllm serve ${model.id} --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice"`; } else { From 98ada8279f1481ddcf1340fa1a47b376903e9122 Mon Sep 17 00:00:00 2001 From: Julien Denize <40604584+juliendenize@users.noreply.github.com> Date: Wed, 23 Jul 2025 13:57:10 +0200 Subject: [PATCH 4/5] Update packages/tasks/src/local-apps.ts Co-authored-by: Lucain --- packages/tasks/src/local-apps.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index 19ce1c3379..aa546089ac 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -221,7 +221,7 @@ curl -X POST "http://localhost:8000/v1/completions" \\ dockerCommand = `# Load and run the model:\ndocker exec -it my_vllm_container bash -c "vllm serve ${model.id} --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice"`; } else { - setup = ["# Install vLLM from pip:", "pip install vllm"].join(""); + setup = ["# Install vLLM from pip:", "pip install vllm"].join("\n"); dockerCommand = `# Load and run the model:\ndocker exec -it my_vllm_container bash -c "vllm serve ${model.id}"`; } From 42971329e0df53f2703b89a530893fa6ef4805f8 Mon Sep 17 00:00:00 2001 From: Julien Denize Date: Wed, 23 Jul 2025 15:09:39 +0200 Subject: [PATCH 5/5] format --- packages/tasks/src/local-apps.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/tasks/src/local-apps.ts b/packages/tasks/src/local-apps.ts index aa546089ac..144c57bdca 100644 --- a/packages/tasks/src/local-apps.ts +++ b/packages/tasks/src/local-apps.ts @@ -210,17 +210,16 @@ curl -X POST "http://localhost:8000/v1/completions" \\ let setup; let dockerCommand; - + if (model.tags.includes("mistral-common")) { setup = [ "# Install vLLM from pip:", "pip install vllm", "# Make sure you have the latest version of mistral-common installed:", - "pip install --upgrade mistral-common" + "pip install --upgrade mistral-common", ].join("\n"); dockerCommand = `# Load and run the model:\ndocker exec -it my_vllm_container bash -c "vllm serve ${model.id} --tokenizer_mode mistral --config_format mistral --load_format mistral --tool-call-parser mistral --enable-auto-tool-choice"`; - } - else { + } else { setup = ["# Install vLLM from pip:", "pip install vllm"].join("\n"); dockerCommand = `# Load and run the model:\ndocker exec -it my_vllm_container bash -c "vllm serve ${model.id}"`; } @@ -244,10 +243,7 @@ curl -X POST "http://localhost:8000/v1/completions" \\ ` vllm/vllm-openai:latest \\`, ` --model ${model.id}`, ].join("\n"), - content: [ - dockerCommand, - runCommand, - ], + content: [dockerCommand, runCommand], }, ]; };