Skip to content

metadata: if LICENSE exist then include a copy of it #8870

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+20 −0
Open

metadata: if LICENSE exist then include a copy of it #8870

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
323bf80
py: if LICENSE exist then include a copy of it
mofosyne Aug 5, 2024
d79e8e1
py: Add LICENSE_CONTENT to metadata override
mofosyne Aug 5, 2024
928aa66
py: remove license_content from metadata heuristics and add additiona…
mofosyne Aug 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions examples/convert_legacy_llama.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -458,6 +458,7 @@ def validate_conversion_to(self, data_type: DataType) -> None:

ModelFormat: TypeAlias = Literal['ggml', 'torch', 'safetensors', 'none']


@dataclass
class ModelPlus:
model: LazyModel
Expand Down Expand Up @@ -810,6 +811,8 @@ def add_meta_model(self, params: Params, metadata: gguf.Metadata | None) -> None
self.gguf.add_license_name(metadata.license_name)
if metadata.license_link is not None:
self.gguf.add_license_link(metadata.license_link)
if metadata.license_content is not None:
self.gguf.add_license_content(metadata.license_content)

if metadata.url is not None:
self.gguf.add_url(metadata.url)
Expand Down
1 change: 1 addition & 0 deletions gguf-py/gguf/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ class General:
LICENSE = "general.license"
LICENSE_NAME = "general.license.name"
LICENSE_LINK = "general.license.link"
LICENSE_CONTENT = "general.license.content"

# Typically represents the converted GGUF repo (Unless native)
URL = "general.url" # Model Website/Paper
Expand Down
3 changes: 3 additions & 0 deletions gguf-py/gguf/gguf_writer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -529,6 +529,9 @@ def add_license_name(self, license: str) -> None:
def add_license_link(self, license: str) -> None:
self.add_string(Keys.General.LICENSE_LINK, license)

def add_license_content(self, license: str) -> None:
self.add_string(Keys.General.LICENSE_CONTENT, license)

def add_url(self, url: str) -> None:
self.add_string(Keys.General.URL, url)

Expand Down
13 changes: 13 additions & 0 deletions gguf-py/gguf/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ class Metadata:
license: Optional[str] = None
license_name: Optional[str] = None
license_link: Optional[str] = None
license_content: Optional[str] = None
base_models: Optional[list[dict]] = None
tags: Optional[list[str]] = None
languages: Optional[list[str]] = None
Expand Down Expand Up @@ -77,6 +78,7 @@ def load(metadata_override_path: Optional[Path] = None, model_path: Optional[Pat
metadata.size_label = metadata_override.get(Keys.General.SIZE_LABEL, metadata.size_label)
metadata.license_name = metadata_override.get(Keys.General.LICENSE_NAME, metadata.license_name)
metadata.license_link = metadata_override.get(Keys.General.LICENSE_LINK, metadata.license_link)
metadata.license_content = metadata_override.get(Keys.General.LICENSE_CONTENT, metadata.license_content)

metadata.url = metadata_override.get(Keys.General.URL, metadata.url)
metadata.doi = metadata_override.get(Keys.General.DOI, metadata.doi)
Expand Down Expand Up @@ -431,6 +433,15 @@ def use_array_model_card_metadata(metadata_key: str, model_card_key: str):
if metadata.size_label is None and size_label is not None:
metadata.size_label = size_label

# Detect LICENSE file and include a copy
#########################################
if isinstance(metadata.license_link, str) and not (metadata.license_link.startswith("http://") or metadata.license_link.startswith("https://")):
if metadata.license_content is None:
standard_license_file_path = Path("LICENSE")
if standard_license_file_path.is_file():
with open(standard_license_file_path, 'r') as file:
metadata.license_content = file.read()

return metadata

def set_gguf_meta_model(self, gguf_writer: gguf.GGUFWriter):
Expand Down Expand Up @@ -463,6 +474,8 @@ def set_gguf_meta_model(self, gguf_writer: gguf.GGUFWriter):
gguf_writer.add_license_name(self.license_name)
if self.license_link is not None:
gguf_writer.add_license_link(self.license_link)
if self.license_content is not None:
gguf_writer.add_license_content(self.license_content)

if self.url is not None:
gguf_writer.add_url(self.url)
Expand Down
Loading