Introduce Q4_0 and Q8_0 quantizations with BF16 delta values

Author: Srihari-mcw

ggml/include/ggml.h

@@ -342,6 +342,7 @@ extern "C" {
 
     // google brain half-precision bfloat16
     typedef struct { uint16_t bits; } ggml_bf16_t;
+    GGML_API ggml_bf16_t ggml_make_bf16(uint16_t val);
     GGML_API ggml_bf16_t ggml_fp32_to_bf16(float);
     GGML_API float       ggml_bf16_to_fp32(ggml_bf16_t);  // consider just doing << 16
     GGML_API void        ggml_bf16_to_fp32_row(const ggml_bf16_t *, float *, int64_t);
@@ -383,6 +384,8 @@ extern "C" {
         GGML_TYPE_F64     = 28,
         GGML_TYPE_IQ1_M   = 29,
         GGML_TYPE_BF16    = 30,
+        GGML_TYPE_Q4_0_B16 = 31,
+        GGML_TYPE_Q8_0_B16 = 32,
         GGML_TYPE_COUNT,
     };
 
@@ -424,6 +427,8 @@ extern "C" {
         GGML_FTYPE_MOSTLY_IQ4_XS  = 22, // except 1d tensors
         GGML_FTYPE_MOSTLY_IQ1_M   = 23, // except 1d tensors
         GGML_FTYPE_MOSTLY_BF16    = 24, // except 1d tensors
+        GGML_FTYPE_MOSTLY_Q4_0_B16 = 25, // except 1d tensors
+        GGML_FTYPE_MOSTLY_Q8_0_B16 = 26, // except 1d tensors
     };
 
     // available tensor operations:

ggml/src/ggml-impl.h

@@ -20,11 +20,13 @@
 #if defined(_MSC_VER)
 
 #define m512bh(p) p
+#define m128bh(p) p
 #define m512i(p) p
 
 #else
 
 #define m512bh(p) (__m512bh)(p)
+#define m128bh(p) (__m128bh)(p)
 #define m512i(p) (__m512i)(p)
 
 #endif