Skip to content

update to support descriptions with file links #8

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 7 commits into from
Jul 3, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 4 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,8 @@ plugins:
markdown_description: Long description of my project.
sections:
Usage documentation:
- file1.md
- file2.md
- file1.md: Description of file1
- file2.md # Descriptions are optional.
```

The resulting `/llms.txt` file will be available at the root of your documentation. With the previous example, it will be accessible at https://myproject.com/llms.txt and will contain the following:
Expand All @@ -46,7 +46,7 @@ Long description of my project.

## Usage documentation

- [File1 title](https://myproject.com/file1.md)
- [File1 title](https://myproject.com/file1.md): Description of file1
- [File2 title](https://myproject.com/file2.md)
```

Expand All @@ -59,7 +59,7 @@ plugins:
- llmstxt:
sections:
Usage documentation:
- index.md
- index.md: Main documentation page
- usage/*.md
```

Expand Down
10 changes: 9 additions & 1 deletion src/mkdocs_llmstxt/_internal/config.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,4 +13,12 @@ class _PluginConfig(BaseConfig):
preprocess = mkconf.Optional(mkconf.File(exists=True))
markdown_description = mkconf.Optional(mkconf.Type(str))
full_output = mkconf.Optional(mkconf.Type(str))
sections = mkconf.DictOfItems(mkconf.ListOfItems(mkconf.Type(str)))
sections = mkconf.DictOfItems(
# Each list item can either be:
#
# - a string representing the source file path (possibly with glob patterns)
# - a mapping where the single key is the file path and the value is its description.
#
# We therefore accept both `str` and `dict` values.
mkconf.ListOfItems(mkconf.Type((str, dict))),
)
40 changes: 26 additions & 14 deletions src/mkdocs_llmstxt/_internal/plugin.py
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ class _MDPageInfo(NamedTuple):
path_md: Path
md_url: str
content: str
description: str


class MkdocsLLMsTxtPlugin(BasePlugin[_PluginConfig]):
Expand All @@ -56,13 +57,21 @@ class MkdocsLLMsTxtPlugin(BasePlugin[_PluginConfig]):
md_pages: dict[str, list[_MDPageInfo]]
"""Dictionary mapping section names to a list of page infos."""

def _expand_inputs(self, inputs: list[str], page_uris: list[str]) -> list[str]:
expanded: list[str] = []
for input_file in inputs:
_sections: dict[str, dict[str, str]]

def _expand_inputs(self, inputs: list[str | dict[str, str]], page_uris: list[str]) -> dict[str, str]:
expanded: dict[str, str] = {}
for input_item in inputs:
if isinstance(input_item, dict):
input_file, description = next(iter(input_item.items()))
else:
input_file = input_item
description = ""
if "*" in input_file:
expanded.extend(fnmatch.filter(page_uris, input_file))
for match in fnmatch.filter(page_uris, input_file):
expanded[match] = description
else:
expanded.append(input_file)
expanded[input_file] = description
return expanded

def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None:
Expand All @@ -81,6 +90,7 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None:
if config.site_url is None:
raise ValueError("'site_url' must be set in the MkDocs configuration to be used with the 'llmstxt' plugin")
self.mkdocs_config = config

# A `defaultdict` could be used, but we need to retain the same order between `config.sections` and `md_pages`
# (which wouldn't be guaranteed when filling `md_pages` in `on_page_content()`).
self.md_pages = {section: [] for section in self.config.sections}
Expand All @@ -100,10 +110,10 @@ def on_files(self, files: Files, *, config: MkDocsConfig) -> Files | None: # no
Modified collection or none.
"""
page_uris = list(files.src_uris)

for section_name, file_list in list(self.config.sections.items()):
self.config.sections[section_name] = self._expand_inputs(file_list, page_uris=page_uris)

self._sections = {
section_name: self._expand_inputs(file_list, page_uris=page_uris) # type: ignore[arg-type]
for section_name, file_list in self.config.sections.items()
}
return files

def on_page_content(self, html: str, *, page: Page, **kwargs: Any) -> str | None: # noqa: ARG002
Expand All @@ -115,8 +125,9 @@ def on_page_content(self, html: str, *, page: Page, **kwargs: Any) -> str | None
html: The rendered HTML.
page: The page object.
"""
for section_name, file_list in self.config.sections.items():
if page.file.src_uri in file_list:
src_uri = page.file.src_uri
for section_name, files in self._sections.items():
if src_uri in files:
path_md = Path(page.file.abs_dest_path).with_suffix(".md")
page_md = _generate_page_markdown(
html,
Expand All @@ -138,10 +149,11 @@ def on_page_content(self, html: str, *, page: Page, **kwargs: Any) -> str | None

self.md_pages[section_name].append(
_MDPageInfo(
title=page.title if page.title is not None else page.file.src_uri,
title=page.title if page.title is not None else src_uri,
path_md=path_md,
md_url=md_url,
content=page_md,
description=files[src_uri],
),
)

Expand Down Expand Up @@ -169,10 +181,10 @@ def on_post_build(self, *, config: MkDocsConfig, **kwargs: Any) -> None: # noqa

for section_name, file_list in self.md_pages.items():
markdown += f"## {section_name}\n\n"
for page_title, path_md, md_url, content in file_list:
for page_title, path_md, md_url, content, desc in file_list:
path_md.write_text(content, encoding="utf8")
_logger.debug(f"Generated MD file to {path_md}")
markdown += f"- [{page_title}]({md_url})\n"
markdown += f"- [{page_title}]({md_url}){(': ' + desc) if desc else ''}\n"
markdown += "\n"

output_file.write_text(markdown, encoding="utf8")
Expand Down
47 changes: 47 additions & 0 deletions tests/conftest.py
Original file line number Diff line number Diff line change
@@ -1 +1,48 @@
"""Configuration for the pytest test suite."""

from __future__ import annotations

from pathlib import Path
from typing import TYPE_CHECKING

import pytest
from mkdocs.config.defaults import MkDocsConfig

if TYPE_CHECKING:
from mkdocs_llmstxt._internal.plugin import MkdocsLLMsTxtPlugin


@pytest.fixture(name="mkdocs_conf")
def fixture_mkdocs_conf(request: pytest.FixtureRequest, tmp_path: Path) -> MkDocsConfig:
"""Yield a MkDocs configuration object."""
while hasattr(request, "_parent_request") and hasattr(request._parent_request, "_parent_request"):
request = request._parent_request
params = getattr(request, "param", {})
config = params.get("config", {})
pages = params.get("pages", {})
conf = MkDocsConfig()
conf.load_dict(
{
"site_name": "Test Project",
"site_url": "https://example.org/",
"site_dir": str(tmp_path / "site"),
"docs_dir": str(tmp_path / "docs"),
**config,
},
)
Path(conf.docs_dir).mkdir(exist_ok=True)
for page, content in pages.items():
page_file = Path(conf.docs_dir, page)
page_file.parent.mkdir(exist_ok=True)
page_file.write_text(content)
assert conf.validate() == ([], [])
if "toc" not in conf.markdown_extensions:
# Guaranteed to be added by MkDocs.
conf.markdown_extensions.insert(0, "toc")
return conf


@pytest.fixture(name="plugin")
def fixture_plugin(mkdocs_conf: MkDocsConfig) -> MkdocsLLMsTxtPlugin:
"""Return a plugin instance."""
return mkdocs_conf.plugins["llmstxt"] # type: ignore[return-value]
58 changes: 52 additions & 6 deletions tests/test_plugin.py
Original file line number Diff line number Diff line change
@@ -1,11 +1,57 @@
"""Tests for the plugin."""

from pathlib import Path

import pytest
from duty.tools import mkdocs
from mkdocs.commands.build import build
from mkdocs.config.defaults import MkDocsConfig


@pytest.mark.parametrize(
"mkdocs_conf",
[
{
"config": {
"plugins": [
{
"llmstxt": {
"full_output": "llms-full.txt",
"sections": {
"Index": ["index.md"],
"Usage": [{"page1.md": "Some usage docs."}],
},
},
},
],
},
"pages": {
"index.md": "# Hello world",
"page1.md": "# Usage\n\nSome paragraph.",
},
},
],
indirect=["mkdocs_conf"],
)
def test_plugin(mkdocs_conf: MkDocsConfig) -> None:
"""Test that page descriptions are correctly handled and included in output."""
build(config=mkdocs_conf)

llmstxt = Path(mkdocs_conf.site_dir, "llms.txt")
assert llmstxt.exists()
llmstxt_content = llmstxt.read_text()
assert "Some usage docs." in llmstxt_content
assert "Some paragraph." not in llmstxt_content

llmsfulltxt = Path(mkdocs_conf.site_dir, "llms-full.txt")
assert llmsfulltxt.exists()
llmsfulltxt_content = llmsfulltxt.read_text()
assert "Some usage docs." not in llmsfulltxt_content
assert "Some paragraph." in llmsfulltxt_content

indexmd = Path(mkdocs_conf.site_dir, "index.md")
assert indexmd.exists()
assert "Hello world" in indexmd.read_text()

def test_plugin() -> None:
"""Run the plugin."""
with pytest.raises(expected_exception=SystemExit) as exc:
mkdocs.build()()
assert exc.value.code == 0
page1md = Path(mkdocs_conf.site_dir, "page1/index.md")
assert page1md.exists()
assert "Some paragraph." in page1md.read_text()
Loading