mamba : fix vocab size problems with official models

The perplexity was waaaay to high for models with a non-round vocab size.
Not sure why, but it needed to be fixed in the metadata.

Note that this breaks existing GGUF-converted Mamba models,
but **only if** the vocab size was not already rounded.

Author: compilade

convert-hf-to-gguf.py

@@ -1768,6 +1768,13 @@ def get_tensors(self):
 
 
 class MambaModel(Model):
+    def set_vocab(self):
+        vocab_size = self.hparams["vocab_size"];
+        # Round vocab size to next multiple of 8
+        pad_vocab = self.hparams.get("pad_vocab_size_multiple", 8);
+        self.hparams["vocab_size"] = ((vocab_size + (pad_vocab - 1)) // pad_vocab) * pad_vocab
+        return self._set_vocab_gpt2()
+
     def set_gguf_parameters(self):
         d_model = self.hparams["d_model"]
         d_inner = self.hparams.get("d_inner", 2 * d_model)

llama.cpp

@@ -4426,15 +4426,13 @@ static bool llm_load_tensors(
                     // FIXME: ceiling instead of floor
                     const int64_t dt_rank = n_embd / 16;
                     GGML_ASSERT(2 * n_embd == d_inner);
-                    // round up the vocab size to the next multiple of 8
-                    const int64_t rounded_vocab = (n_vocab + 7) & -8;
 
-                    model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, rounded_vocab});
+                    model.tok_embd = ml.create_tensor(ctx_input, tn(LLM_TENSOR_TOKEN_EMBD, "weight"), {n_embd, n_vocab});
 
                     // output
                     {
                         model.output_norm   = ml.create_tensor(ctx_output,       tn(LLM_TENSOR_OUTPUT_NORM, "weight"), {n_embd});
-                        model.output        = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, rounded_vocab});
+                        model.output        = ml.create_tensor(ctx_output_split, tn(LLM_TENSOR_OUTPUT,      "weight"), {n_embd, n_vocab});
                     }
 
                     for (int i = 0; i < n_layer; ++i) {