add file data to kv as STRING

Author: katsu560

gguf-py/gguf/constants.py

@@ -2,7 +2,6 @@
 
 from enum import Enum, IntEnum, auto
 from typing import Any
-from dataclasses import dataclass
 
 #
 # constants
@@ -13,15 +12,6 @@
 GGUF_DEFAULT_ALIGNMENT = 32
 GGML_QUANT_VERSION     = 2  # GGML_QNT_VERSION from ggml.h
 
-#
-# datatype
-#
-
-@dataclass
-class NamedObject:
-    name: str
-    obj: bytes[Any]
- 
 #
 # metadata keys
 #
@@ -41,8 +31,7 @@ class General:
         SOURCE_URL           = "general.source.url"
         SOURCE_HF_REPO       = "general.source.huggingface.repository"
         FILE_TYPE            = "general.file_type"
-        NAMEDOBJECT          = "general.namedobject"
-        CONNECT              = "."
+        FILE_MARK            = "/"
 
     class LLM:
         VOCAB_SIZE            = "{arch}.vocab_size"
@@ -913,14 +902,11 @@ class GGUFValueType(IntEnum):
     UINT64  = 10
     INT64   = 11
     FLOAT64 = 12
-    NAMEDOBJECT = 13
 
     @staticmethod
     def get_type(val: Any) -> GGUFValueType:
         if isinstance(val, (str, bytes, bytearray)):
             return GGUFValueType.STRING
-        elif isinstance(val, (str, bytes, bytearray)):
-            return GGUFValueType.NAMEDOBJECT
         elif isinstance(val, list):
             return GGUFValueType.ARRAY
         elif isinstance(val, float):

gguf-py/gguf/gguf_reader.py

@@ -170,15 +170,6 @@ def _get_field_parts(
             sparts: list[npt.NDArray[Any]] = list(self._get_str(offs))
             size = sum(int(part.nbytes) for part in sparts)
             return size, sparts, [1], types
-        # Handle namedobjects.
-        if gtype == GGUFValueType.NAMEDOBJECT:
-            nparts: list[npt.NDArray[Any]] = list(self._get_str(offs))
-            nsize = sum(int(part.nbytes) for part in nparts)
-            oparts: list[npt.NDArray[Any]] = list(self._get_obj(offs + nsize))
-            osize = sum(int(part.nbytes) for part in oparts)
-            nosize = nsize + osize
-            noparts: list[npt.NDArray[Any]] = list((nparts[0],nparts[1],oparts[0],oparts[1]))
-            return nosize, noparts, [4], types
         # Check if it's a simple scalar type.
         nptype = self.gguf_scalar_to_np.get(gtype)
         if nptype is not None:

gguf-py/gguf/gguf_writer.py

@@ -75,8 +75,6 @@ def __init__(
             "Big" if self.endianess == GGUFEndian.BIG else "Little",
         ))
         self.state = WriterState.EMPTY
-        # namedobject
-        self.namedobject_count = 0 
 
         self.add_architecture()
 
@@ -167,33 +165,19 @@ def add_array(self, key: str, val: Sequence[Any]) -> None:
         self.add_key(key)
         self.add_val(val, GGUFValueType.ARRAY)
 
-    def add_namedobject(self, key: str, val: bytes[Any], name: str, array: NamedObject[Any] | None = None) -> None:
-        # array: False: add as each namedobject, True: add as each element of array of namedobject
-        if not name:
-            raise ValueError("Need name for namedobject")
+    def add_object(self, key: str, val: bytes[Any]) -> None:
         if not val:
-            raise ValueError("Need val for namedobject")
+            raise ValueError("Need val for object")
 
-        if array is None:
-            self.namedobject_count += 1
-            key = Keys.General.NAMEDOBJECT + Keys.General.CONNECT + str(self.namedobject_count)
-            self.add_key(key)
-            self.add_val(val, GGUFValueType.NAMEDOBJECT, name=name)
-        else:
-            # if array, val and name is dammy
-            key = Keys.General.NAMEDOBJECT
-            self.add_key(key)
-            vtype = GGUFValueType.ARRAY
-            self.kv_data += self._pack("I", vtype)
-            self.kv_data_count += 1
-            ltype = GGUFValueType.NAMEDOBJECT
-            self.kv_data += self._pack("I", ltype)
-            self.kv_data += self._pack("Q", len(array))
-            for item in array:
-                self.add_val(item.obj, GGUFValueType.NAMEDOBJECT, add_vtype=False, name=item.name)
+        # store object as STRING
+        self.add_key(key)
+        self.kv_data += self._pack("I", GGUFValueType.STRING)
+        self.kv_data_count += 1
+        self.kv_data += self._pack("Q", len(val))
+        self.kv_data += val
 
 
-    def add_val(self, val: Any, vtype: GGUFValueType | None = None, add_vtype: bool = True, name: str | None = None) -> None:
+    def add_val(self, val: Any, vtype: GGUFValueType | None = None, add_vtype: bool = True) -> None:
         if vtype is None:
             vtype = GGUFValueType.get_type(val)
 
@@ -208,12 +192,6 @@ def add_val(self, val: Any, vtype: GGUFValueType | None = None, add_vtype: bool
             encoded_val = val.encode("utf-8") if isinstance(val, str) else val
             self.kv_data += self._pack("Q", len(encoded_val))
             self.kv_data += encoded_val
-        elif vtype == GGUFValueType.NAMEDOBJECT:
-            encoded_name = name.encode("utf8") if isinstance(name, str) else name
-            self.kv_data += self._pack("Q", len(encoded_name))
-            self.kv_data += encoded_name
-            self.kv_data += self._pack("Q", len(val))
-            self.kv_data += val
         elif vtype == GGUFValueType.ARRAY and isinstance(val, Sequence) and val:
             ltype = GGUFValueType.get_type(val[0])
             if not all(GGUFValueType.get_type(i) is ltype for i in val[1:]):

gguf-py/scripts/gguf-addfile.py

@@ -17,7 +17,7 @@
     #print("add path", str(Path(__file__).parent.parent))
     sys.path.insert(0, str(Path(__file__).parent.parent))
 
-from gguf import GGUFReader, GGUFWriter, ReaderField, GGUFEndian, GGUFValueType, Keys, NamedObject  # noqa: E402
+from gguf import GGUFReader, GGUFWriter, ReaderField, GGUFEndian, GGUFValueType, Keys  # noqa: E402
 
 logger = logging.getLogger("gguf-addfile")
 
@@ -49,10 +49,10 @@ def dump_metadata(reader: GGUFReader, args: argparse.Namespace) -> None:
         if len(field.types) == 1:
             curr_type = field.types[0]
             if curr_type == GGUFValueType.STRING:
-                print(' = {0}'.format(repr(str(bytes(field.parts[-1]), encoding='utf8')[:60])), end = '')
-            elif curr_type == GGUFValueType.NAMEDOBJECT:
-                print(' = {0}'.format(repr(str(bytes(field.parts[4]), encoding='utf8')[:60])), end = '')
-                print(', {0}'.format(int(field.parts[5]))[:20], end = '')
+                if not field.name[0] == Keys.General.FILE_MARK:
+                  print(' = {0}'.format(repr(str(bytes(field.parts[-1]), encoding='utf8')[:60])), end = '')
+                else:
+                  print(' = binary data', end = '')
             elif field.types[0] in reader.gguf_scalar_to_np:
                 print(' = {0}'.format(field.parts[-1][0]), end = '')
         print()
@@ -88,16 +88,17 @@ def dump_metadata_json(reader: GGUFReader, args: argparse.Namespace) -> None:
                 continue
             itype = field.types[-1]
             if itype == GGUFValueType.STRING:
-                curr["value"] = [str(bytes(field.parts[idx]), encoding="utf-8") for idx in field.data]
-            elif itype == GGUFValueType.NAMEDOBJECT:
-                curr["value"] = [str(bytes(field.parts[idx]), encoding="utf-8") for idx in field.data]
+                if not field.name[0] == Keys.General.FILE_MARK:
+                  curr["value"] = [str(bytes(field.parts[idx]), encoding="utf-8") for idx in field.data]
+                else:
+                  curr["value"] = [bytes(field.parts[idx]) for idx in field.data]
             else:
                 curr["value"] = [pv for idx in field.data for pv in field.parts[idx].tolist()]
         elif field.types[0] == GGUFValueType.STRING:
-            curr["value"] = str(bytes(field.parts[-1]), encoding="utf-8")
-        elif field.types[0] == GGUFValueType.NAMEDOBJECT:
-            curr["value"] = str(bytes(field.parts[4]), encoding="utf-8")
-            curr["value"] = int(field.parts[5])
+            if not field.name[0] == Keys.General.FILE_MARK:
+                curr["value"] = str(bytes(field.parts[-1]), encoding="utf-8")
+            else:
+                curr["value"] = bytes(field.parts[-1])
         else:
             curr["value"] = field.parts[-1].tolist()[0]
     if not args.no_tensors:
@@ -135,15 +136,17 @@ def decode_field(field: ReaderField) -> Any:
             sub_type = field.types[-1]
 
             if sub_type == GGUFValueType.STRING:
-                return [str(bytes(field.parts[idx]), encoding='utf8') for idx in field.data]
-            elif sub_type == GGUFValueType.NAMEDOBJECT:
-                return [str(bytes(field.parts[idx]), encoding='utf8') for idx in field.data]
+                if not field.name[0] == Keys.General.FILE_MARK:
+                    return [str(bytes(field.parts[idx]), encoding='utf8') for idx in field.data]
+                else:
+                    return [bytes(field.parts[idx]) for idx in field.data]
             else:
                 return [pv for idx in field.data for pv in field.parts[idx].tolist()]
         if main_type == GGUFValueType.STRING:
-            return str(bytes(field.parts[-1]), encoding='utf8')
-        elif main_type == GGUFValueType.NAMEDOBJECT:
-            return str(bytes(field.parts[4]), encoding='utf8')
+            if not field.name[0] == Keys.General.FILE_MARK:
+                return str(bytes(field.parts[-1]), encoding='utf8')
+            else:
+                return bytes(field.parts[-1])
         else:
             return field.parts[-1][0]
 
@@ -156,7 +159,7 @@ def get_field_data(reader: GGUFReader, key: str) -> Any:
     return decode_field(field)
 
 
-def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: Mapping[str, str], array: NamedObject[Any] | None = None) -> None:
+def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new_metadata: Mapping[str, str]) -> None:
     for field in reader.fields.values():
         # Suppress virtual fields and fields written by GGUFWriter
         if field.name == Keys.General.ARCHITECTURE or field.name.startswith('GGUF.'):
@@ -186,18 +189,11 @@ def copy_with_new_metadata(reader: gguf.GGUFReader, writer: gguf.GGUFWriter, new
         writer.add_chat_template(new_metadata[Keys.Tokenizer.CHAT_TEMPLATE])
         del new_metadata[Keys.Tokenizer.CHAT_TEMPLATE]
 
-    if array is None:
-        for key, name in new_metadata.items():
-            logger.debug(f'Adding {key}: {name}')
-            # named object
-            with open(name, "rb") as f:
-                val = f.read()
-                writer.add_namedobject(key, val, name)
-    else:
-        for key, name in new_metadata.items():
-            logger.debug(f'Adding array {key}: {name}')
-            # named object
-            writer.add_namedobject(key, 'val', name, array=array)
+    for key, name in new_metadata.items():
+        logger.debug(f'Adding {key}: {name}')
+        with open(name, "rb") as f:
+            val = f.read()
+            writer.add_object(key, val)
 
     for tensor in reader.tensors:
         # Dimensions are written in reverse order, so flip them first
@@ -219,7 +215,6 @@ def main() -> None:
     parser.add_argument("input",        type=str,            help="GGUF format model input filename")
     parser.add_argument("output",       type=str,            help="GGUF format model output filename")
     parser.add_argument("addfiles",     type=str, nargs='+', help="add filenames ...")
-    parser.add_argument("--array",      action="store_true", help="add files to namedobject array")
     parser.add_argument("--no-tensors", action="store_true", help="Don't dump tensor metadata")
     parser.add_argument("--json",       action="store_true", help="Produce JSON output")
     parser.add_argument("--json-array", action="store_true", help="Include full array values in JSON output (long)")
@@ -242,27 +237,12 @@ def main() -> None:
 
     logger.info(f'* Adding: {args.addfiles}')
     new_metadata = {}
-    count = 0
-    if args.array is False:
-        for path in args.addfiles:
-            count += 1
-            key = Keys.General.NAMEDOBJECT + Keys.General.CONNECT + str(count)
-            new_metadata[key] = path
-            logger.info(f'* Adding: {key} = {path}')
-        copy_with_new_metadata(reader, writer, new_metadata)
-    else:
-        key = Keys.General.NAMEDOBJECT
-        # array is dummy
-        new_metadata[key] = 'array'
-        files = []
-        for path in args.addfiles:
-            with open(path, "rb") as f:
-                val = f.read()
-                #print(f'files[{count}] = {path}')
-                files.append(NamedObject(path, val))
-            logger.info(f'* Adding: {key}[{count}] = {path}')
-            count += 1
-        copy_with_new_metadata(reader, writer, new_metadata, array=files)
+    for path in args.addfiles:
+        # add FILE_MARK to key
+        key = Keys.General.FILE_MARK + path
+        new_metadata[key] = path
+        logger.info(f'* Adding: {key} = {path}')
+    copy_with_new_metadata(reader, writer, new_metadata)
 
     if args.json:
         dump_metadata_json(reader, args)