gguf : embed files to gguf

Author: katsu560

gguf-py/gguf/constants.py

@@ -2,6 +2,7 @@
 
 from enum import Enum, IntEnum, auto
 from typing import Any
+from dataclasses import dataclass
 
 #
 # constants
@@ -12,6 +13,15 @@
 GGUF_DEFAULT_ALIGNMENT = 32
 GGML_QUANT_VERSION     = 2  # GGML_QNT_VERSION from ggml.h
 
+#
+# datatype
+#
+
+@dataclass
+class NamedObject:
+    name: str
+    obj: bytes[Any]
+ 
 #
 # metadata keys
 #
@@ -31,6 +41,8 @@ class General:
         SOURCE_URL           = "general.source.url"
         SOURCE_HF_REPO       = "general.source.huggingface.repository"
         FILE_TYPE            = "general.file_type"
+        NAMEDOBJECT          = "general.namedobject"
+        CONNECT              = "."
 
     class LLM:
         VOCAB_SIZE            = "{arch}.vocab_size"
@@ -901,11 +913,14 @@ class GGUFValueType(IntEnum):
     UINT64  = 10
     INT64   = 11
     FLOAT64 = 12
+    NAMEDOBJECT = 13
 
     @staticmethod
     def get_type(val: Any) -> GGUFValueType:
         if isinstance(val, (str, bytes, bytearray)):
             return GGUFValueType.STRING
+        elif isinstance(val, (str, bytes, bytearray)):
+            return GGUFValueType.NAMEDOBJECT
         elif isinstance(val, list):
             return GGUFValueType.ARRAY
         elif isinstance(val, float):

gguf-py/gguf/gguf_reader.py

@@ -154,6 +154,10 @@ def _get_str(self, offset: int) -> tuple[npt.NDArray[np.uint64], npt.NDArray[np.
         slen = self._get(offset, np.uint64)
         return slen, self._get(offset + 8, np.uint8, slen[0])
 
+    def _get_obj(self, offset: int) -> tuple[npt.NDArray[np.uint64], npt.NDArray[np.uint8]]:
+        olen = self._get(offset, np.uint64)
+        return olen, self._get(offset + 8, np.uint8, olen[0])
+
     def _get_field_parts(
         self, orig_offs: int, raw_type: int,
     ) -> tuple[int, list[npt.NDArray[Any]], list[int], list[GGUFValueType]]:
@@ -166,6 +170,15 @@ def _get_field_parts(
             sparts: list[npt.NDArray[Any]] = list(self._get_str(offs))
             size = sum(int(part.nbytes) for part in sparts)
             return size, sparts, [1], types
+        # Handle namedobjects.
+        if gtype == GGUFValueType.NAMEDOBJECT:
+            nparts: list[npt.NDArray[Any]] = list(self._get_str(offs))
+            nsize = sum(int(part.nbytes) for part in nparts)
+            oparts: list[npt.NDArray[Any]] = list(self._get_obj(offs + nsize))
+            osize = sum(int(part.nbytes) for part in oparts)
+            nosize = nsize + osize
+            noparts: list[npt.NDArray[Any]] = list((nparts[0],nparts[1],oparts[0],oparts[1]))
+            return nosize, noparts, [4], types
         # Check if it's a simple scalar type.
         nptype = self.gguf_scalar_to_np.get(gtype)
         if nptype is not None:

gguf-py/gguf/gguf_writer.py

@@ -75,6 +75,8 @@ def __init__(
             "Big" if self.endianess == GGUFEndian.BIG else "Little",
         ))
         self.state = WriterState.EMPTY
+        # namedobject
+        self.namedobject_count = 0 
 
         self.add_architecture()
 
@@ -165,7 +167,33 @@ def add_array(self, key: str, val: Sequence[Any]) -> None:
         self.add_key(key)
         self.add_val(val, GGUFValueType.ARRAY)
 
-    def add_val(self, val: Any, vtype: GGUFValueType | None = None, add_vtype: bool = True) -> None:
+    def add_namedobject(self, key: str, val: bytes[Any], name: str, array: NamedObject[Any] | None = None) -> None:
+        # array: False: add as each namedobject, True: add as each element of array of namedobject
+        if not name:
+            raise ValueError("Need name for namedobject")
+        if not val:
+            raise ValueError("Need val for namedobject")
+
+        if array is None:
+            self.namedobject_count += 1
+            key = Keys.General.NAMEDOBJECT + Keys.General.CONNECT + str(self.namedobject_count)
+            self.add_key(key)
+            self.add_val(val, GGUFValueType.NAMEDOBJECT, name=name)
+        else:
+            # if array, val and name is dammy
+            key = Keys.General.NAMEDOBJECT
+            self.add_key(key)
+            vtype = GGUFValueType.ARRAY
+            self.kv_data += self._pack("I", vtype)
+            self.kv_data_count += 1
+            ltype = GGUFValueType.NAMEDOBJECT
+            self.kv_data += self._pack("I", ltype)
+            self.kv_data += self._pack("Q", len(array))
+            for item in array:
+                self.add_val(item.obj, GGUFValueType.NAMEDOBJECT, add_vtype=False, name=item.name)
+
+
+    def add_val(self, val: Any, vtype: GGUFValueType | None = None, add_vtype: bool = True, name: str | None = None) -> None:
         if vtype is None:
             vtype = GGUFValueType.get_type(val)
 
@@ -180,6 +208,12 @@ def add_val(self, val: Any, vtype: GGUFValueType | None = None, add_vtype: bool
             encoded_val = val.encode("utf-8") if isinstance(val, str) else val
             self.kv_data += self._pack("Q", len(encoded_val))
             self.kv_data += encoded_val
+        elif vtype == GGUFValueType.NAMEDOBJECT:
+            encoded_name = name.encode("utf8") if isinstance(name, str) else name
+            self.kv_data += self._pack("Q", len(encoded_name))
+            self.kv_data += encoded_name
+            self.kv_data += self._pack("Q", len(val))
+            self.kv_data += val
         elif vtype == GGUFValueType.ARRAY and isinstance(val, Sequence) and val:
             ltype = GGUFValueType.get_type(val[0])
             if not all(GGUFValueType.get_type(i) is ltype for i in val[1:]):