Skip to content

Commit 82b2404

Browse files
compiladecompilade
committed
Merge branch 'master' into compilade/bitnet-ternary
2 parents 69f7726 + 06943a6 commit 82b2404

File tree

22 files changed

+75
-58
lines changed

22 files changed

+75
-58
lines changed

‎.github/workflows/bench.yml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -129,6 +129,8 @@ jobs:
129129
130130
- name: Server bench
131131
id: server_bench
132+
env:
133+
HEAD_REF: ${{ github.head_ref || github.ref_name }}
132134
run: |
133135
set -eux
134136
@@ -137,7 +139,7 @@ jobs:
137139
python bench.py \
138140
--runner-label ${{ env.RUNNER_LABEL }} \
139141
--name ${{ github.job }} \
140-
--branch ${{ github.head_ref || github.ref_name }} \
142+
--branch $HEAD_REF \
141143
--commit ${{ github.event.inputs.sha || github.event.pull_request.head.sha || github.sha }} \
142144
--scenario script.js \
143145
--duration ${{ github.event.inputs.duration || env.DURATION }} \

‎.github/workflows/build.yml

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -47,7 +47,7 @@ jobs:
4747
sysctl -a
4848
mkdir build
4949
cd build
50-
cmake -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL_EMBED_LIBRARY=ON -DLLAMA_CURL=ON -DBUILD_SHARED_LIBS=OFF ..
50+
cmake -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL_EMBED_LIBRARY=ON -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF ..
5151
cmake --build . --config Release -j $(sysctl -n hw.logicalcpu)
5252
5353
- name: Test
@@ -105,7 +105,7 @@ jobs:
105105
sysctl -a
106106
# Metal is disabled due to intermittent failures with Github runners not having a GPU:
107107
# https://github.com/ggerganov/llama.cpp/actions/runs/8635935781/job/23674807267#step:5:2313
108-
cmake -B build -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL=OFF -DLLAMA_CURL=ON -DBUILD_SHARED_LIBS=OFF
108+
cmake -B build -DLLAMA_FATAL_WARNINGS=ON -DGGML_METAL=OFF -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF
109109
cmake --build build --config Release -j $(sysctl -n hw.logicalcpu)
110110
111111
- name: Test
@@ -222,7 +222,7 @@ jobs:
222222
run: |
223223
mkdir build
224224
cd build
225-
cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=ON -DBUILD_SHARED_LIBS=OFF
225+
cmake .. -DLLAMA_FATAL_WARNINGS=ON -DLLAMA_CURL=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=OFF
226226
cmake --build . --config Release -j $(nproc)
227227
228228
- name: Test
@@ -696,22 +696,20 @@ jobs:
696696
strategy:
697697
matrix:
698698
include:
699-
- build: 'rpc-x64'
700-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=ON'
701699
- build: 'noavx-x64'
702-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DBUILD_SHARED_LIBS=ON'
700+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_AVX=OFF -DGGML_AVX2=OFF -DGGML_FMA=OFF -DBUILD_SHARED_LIBS=ON'
703701
- build: 'avx2-x64'
704-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON'
702+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DBUILD_SHARED_LIBS=ON'
705703
- build: 'avx-x64'
706-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_AVX2=OFF -DBUILD_SHARED_LIBS=ON'
704+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_AVX2=OFF -DBUILD_SHARED_LIBS=ON'
707705
- build: 'avx512-x64'
708-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
706+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_AVX512=ON -DBUILD_SHARED_LIBS=ON'
709707
- build: 'openblas-x64'
710-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_BLAS=ON -DBUILD_SHARED_LIBS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
708+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_BLAS=ON -DBUILD_SHARED_LIBS=ON -DGGML_BLAS_VENDOR=OpenBLAS -DBLAS_INCLUDE_DIRS="$env:RUNNER_TEMP/openblas/include" -DBLAS_LIBRARIES="$env:RUNNER_TEMP/openblas/lib/openblas.lib"'
711709
- build: 'kompute-x64'
712-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DBUILD_SHARED_LIBS=ON'
710+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_KOMPUTE=ON -DKOMPUTE_OPT_DISABLE_VULKAN_VERSION_CHECK=ON -DBUILD_SHARED_LIBS=ON'
713711
- build: 'vulkan-x64'
714-
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_VULKAN=ON -DBUILD_SHARED_LIBS=ON'
712+
defines: '-DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DGGML_RPC=ON -DGGML_VULKAN=ON -DBUILD_SHARED_LIBS=ON'
715713
- build: 'llvm-arm64'
716714
defines: '-G "Ninja Multi-Config" -D CMAKE_TOOLCHAIN_FILE=cmake/arm64-windows-llvm.cmake -DGGML_NATIVE=OFF -DLLAMA_BUILD_SERVER=ON -DBUILD_SHARED_LIBS=ON'
717715
- build: 'msvc-arm64'

‎.github/workflows/python-check-requirements.yml

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,13 @@ on:
66
- '.github/workflows/python-check-requirements.yml'
77
- 'scripts/check-requirements.sh'
88
- 'convert*.py'
9-
- 'requirements.txt'
10-
- 'requirements/*.txt'
9+
- '**/requirements*.txt'
1110
pull_request:
1211
paths:
1312
- '.github/workflows/python-check-requirements.yml'
1413
- 'scripts/check-requirements.sh'
1514
- 'convert*.py'
16-
- 'requirements.txt'
17-
- 'requirements/*.txt'
15+
- '**/requirements*.txt'
1816

1917
concurrency:
2018
group: ${{ github.workflow }}-${{ github.head_ref && github.ref || github.run_id }}

‎README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,10 +186,12 @@ Unless otherwise noted these projects are open-source with permissive licensing:
186186

187187
- [akx/ggify](https://github.com/akx/ggify) – download PyTorch models from HuggingFace Hub and convert them to GGML
188188
- [crashr/gppm](https://github.com/crashr/gppm) – launch llama.cpp instances utilizing NVIDIA Tesla P40 or P100 GPUs with reduced idle power consumption
189+
- [gpustack/gguf-parser](https://github.com/gpustack/gguf-parser-go/tree/main/cmd/gguf-parser) - review/check the GGUF file and estimate the memory usage
189190

190191
**Infrastructure:**
191192

192193
- [Paddler](https://github.com/distantmagic/paddler) - Stateful load balancer custom-tailored for llama.cpp
194+
- [GPUStack](https://github.com/gpustack/gpustack) - Manage GPU clusters for running LLMs
193195

194196
**Games:**
195197
- [Lucy's Labyrinth](https://github.com/MorganRO8/Lucys_Labyrinth) - A simple maze game where agents controlled by an AI model will try to trick you.

‎common/grammar-parser.cpp

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -369,6 +369,9 @@ namespace grammar_parser {
369369
}
370370
// Validate the state to ensure that all rules are defined
371371
for (const auto & rule : state.rules) {
372+
if (rule.empty()) {
373+
throw std::runtime_error("Undefined rule");
374+
}
372375
for (const auto & elem : rule) {
373376
if (elem.type == LLAMA_GRETYPE_RULE_REF) {
374377
// Ensure that the rule at that location exists

‎examples/export-lora/README.md

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ For example:
1717

1818
```bash
1919
./bin/llama-export-lora \
20-
-m open-llama-3b-v2-q8_0.gguf \
21-
-o open-llama-3b-v2-q8_0-english2tokipona-chat.gguf \
22-
--lora lora-open-llama-3b-v2-q8_0-english2tokipona-chat-LATEST.gguf
20+
-m open-llama-3b-v2.gguf \
21+
-o open-llama-3b-v2-english2tokipona-chat.gguf \
22+
--lora lora-open-llama-3b-v2-english2tokipona-chat-LATEST.gguf
2323
```
2424

2525
Multiple LORA adapters can be applied by passing multiple `--lora FNAME` or `--lora-scaled FNAME S` command line parameters:

‎examples/export-lora/export-lora.cpp

Lines changed: 26 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,12 @@
1010

1111
static bool g_verbose = false;
1212

13+
struct tensor_transformation {
14+
struct ggml_tensor * in;
15+
struct ggml_tensor * out;
16+
bool is_copy;
17+
};
18+
1319
static std::string get_kv_str(struct gguf_context * ctx_gguf, const std::string & key){
1420
int id = gguf_find_key(ctx_gguf, key.c_str());
1521
return id < 0 ? "" : std::string(gguf_get_val_str(ctx_gguf, id));
@@ -198,8 +204,7 @@ struct lora_merge_ctx {
198204
}
199205

200206
// mapping base tensor to out tensor (same shape with base, but different type)
201-
// if out_tensor == nullptr, we only copy it
202-
std::vector<std::pair<struct ggml_tensor *, struct ggml_tensor *>> base_to_out_tensors;
207+
std::vector<tensor_transformation> trans;
203208
for (auto & it : base_model.tensors) {
204209
bool t_a = true;
205210
bool t_b = true;
@@ -212,14 +217,22 @@ struct lora_merge_ctx {
212217
// only copy
213218
struct ggml_tensor * cpy_tensor = ggml_dup_tensor(ctx_out_ggml, base_tensor);
214219
ggml_set_name(cpy_tensor, base_tensor->name);
215-
base_to_out_tensors.push_back(std::make_pair(cpy_tensor, nullptr));
220+
trans.push_back({
221+
cpy_tensor,
222+
cpy_tensor,
223+
true,
224+
});
216225
gguf_add_tensor(ctx_out, cpy_tensor);
217226
} else if (t_a && t_b) {
218227
// need merging
219228
struct ggml_tensor * out_tensor = ggml_new_tensor(
220229
ctx_out_ggml, get_out_tensor_type(base_tensor), GGML_MAX_DIMS, base_tensor->ne);
221230
ggml_set_name(out_tensor, base_tensor->name);
222-
base_to_out_tensors.push_back(std::make_pair(base_tensor, out_tensor));
231+
trans.push_back({
232+
base_tensor,
233+
out_tensor,
234+
false,
235+
});
223236
gguf_add_tensor(ctx_out, out_tensor);
224237
} else {
225238
throw std::runtime_error("tensor " + it.first + " missing either lora_a or lora_b");
@@ -234,12 +247,12 @@ struct lora_merge_ctx {
234247

235248
// process base model tensors
236249
size_t n_merged = 0;
237-
for (auto & it : base_to_out_tensors) {
238-
if (it.second != nullptr) {
239-
merge_tensor(it.first, it.second);
250+
for (auto & it : trans) {
251+
if (!it.is_copy) {
252+
merge_tensor(it.in, it.out);
240253
n_merged++;
241254
} else {
242-
copy_tensor(it.first);
255+
copy_tensor(it.in);
243256
}
244257
}
245258

@@ -252,7 +265,7 @@ struct lora_merge_ctx {
252265
}
253266

254267
printf("%s : merged %ld tensors with lora adapters\n", __func__, n_merged);
255-
printf("%s : wrote %ld tensors to output file\n", __func__, base_to_out_tensors.size());
268+
printf("%s : wrote %ld tensors to output file\n", __func__, trans.size());
256269
}
257270

258271
void copy_tensor(struct ggml_tensor * base) {
@@ -285,6 +298,10 @@ struct lora_merge_ctx {
285298
for (size_t i = 0; i < adapters.size(); ++i) {
286299
auto t_a = adapters[i]->get_tensor(name_lora_a);
287300
auto t_b = adapters[i]->get_tensor(name_lora_b);
301+
// TODO: add support for quantized lora
302+
if (ggml_is_quantized(t_a->type) || ggml_is_quantized(t_b->type)) {
303+
throw std::runtime_error("quantized LoRA adapters is not supported, please retry with f16 or f32");
304+
}
288305
inp_a[i] = ggml_dup_tensor(ctx, t_a);
289306
inp_b[i] = ggml_dup_tensor(ctx, t_b);
290307
}

‎examples/llava/requirements.txt

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,4 @@
22
--extra-index-url https://download.pytorch.org/whl/cpu
33
pillow~=10.2.0
44
torch~=2.2.1
5-
torchvision==0.17.1
5+
torchvision~=0.17.1

‎examples/server/server.cpp

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -631,6 +631,7 @@ struct server_context {
631631

632632
bool clean_kv_cache = true;
633633
bool add_bos_token = true;
634+
bool has_eos_token = false;
634635

635636
int32_t n_ctx; // total context for all clients / slots
636637

@@ -693,7 +694,7 @@ struct server_context {
693694
n_ctx = llama_n_ctx(ctx);
694695

695696
add_bos_token = llama_should_add_bos_token(model);
696-
GGML_ASSERT(llama_add_eos_token(model) != 1);
697+
has_eos_token = llama_add_eos_token(model) != 1;
697698

698699
return true;
699700
}
@@ -1031,7 +1032,7 @@ struct server_context {
10311032
{
10321033
slot.sparams.logit_bias.clear();
10331034

1034-
if (json_value(data, "ignore_eos", false)) {
1035+
if (json_value(data, "ignore_eos", false) && has_eos_token) {
10351036
slot.sparams.logit_bias[llama_token_eos(model)] = -INFINITY;
10361037
}
10371038

‎ggml/include/ggml.h

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,8 @@
244244
#define GGML_EXIT_SUCCESS 0
245245
#define GGML_EXIT_ABORTED 1
246246

247+
#define GGML_ROPE_TYPE_NEOX 2
248+
247249
#define GGUF_MAGIC "GGUF"
248250

249251
#define GGUF_VERSION 3
@@ -1455,8 +1457,8 @@ extern "C" {
14551457
struct ggml_tensor * b);
14561458

14571459
// rotary position embedding
1458-
// if mode & 1 == 1, skip n_past elements (NOT SUPPORTED)
1459-
// if mode & 2 == 1, GPT-NeoX style
1460+
// if (mode & 1) - skip n_past elements (NOT SUPPORTED)
1461+
// if (mode & GGML_ROPE_TYPE_NEOX) - GPT-NeoX style
14601462
//
14611463
// b is an int32 vector with size a->ne[2], it contains the positions
14621464
GGML_API struct ggml_tensor * ggml_rope(

0 commit comments

Comments
 (0)