diff --git a/README.md b/README.md index b080569..b3a5ee5 100644 --- a/README.md +++ b/README.md @@ -31,8 +31,8 @@ plugins: markdown_description: Long description of my project. sections: Usage documentation: - - file1.md - - file2.md + - file1.md: Description of file1 + - file2.md # Descriptions are optional. ``` The resulting `/llms.txt` file will be available at the root of your documentation. With the previous example, it will be accessible at https://myproject.com/llms.txt and will contain the following: @@ -46,7 +46,7 @@ Long description of my project. ## Usage documentation -- [File1 title](https://myproject.com/file1.md) +- [File1 title](https://myproject.com/file1.md): Description of file1 - [File2 title](https://myproject.com/file2.md) ``` @@ -59,7 +59,7 @@ plugins: - llmstxt: sections: Usage documentation: - - index.md + - index.md: Main documentation page - usage/*.md ``` diff --git a/src/mkdocs_llmstxt/_internal/config.py b/src/mkdocs_llmstxt/_internal/config.py index a9f5877..c5fef5d 100644 --- a/src/mkdocs_llmstxt/_internal/config.py +++ b/src/mkdocs_llmstxt/_internal/config.py @@ -13,4 +13,12 @@ class _PluginConfig(BaseConfig): preprocess = mkconf.Optional(mkconf.File(exists=True)) markdown_description = mkconf.Optional(mkconf.Type(str)) full_output = mkconf.Optional(mkconf.Type(str)) - sections = mkconf.DictOfItems(mkconf.ListOfItems(mkconf.Type(str))) + sections = mkconf.DictOfItems( + # Each list item can either be: + # + # - a string representing the source file path (possibly with glob patterns) + # - a mapping where the single key is the file path and the value is its description. + # + # We therefore accept both `str` and `dict` values. + mkconf.ListOfItems(mkconf.Type((str, dict))), + ) diff --git a/src/mkdocs_llmstxt/_internal/plugin.py b/src/mkdocs_llmstxt/_internal/plugin.py index e89f2bb..e922484 100644 --- a/src/mkdocs_llmstxt/_internal/plugin.py +++ b/src/mkdocs_llmstxt/_internal/plugin.py @@ -36,6 +36,7 @@ class _MDPageInfo(NamedTuple): path_md: Path md_url: str content: str + description: str class MkdocsLLMsTxtPlugin(BasePlugin[_PluginConfig]): @@ -56,13 +57,21 @@ class MkdocsLLMsTxtPlugin(BasePlugin[_PluginConfig]): md_pages: dict[str, list[_MDPageInfo]] """Dictionary mapping section names to a list of page infos.""" - def _expand_inputs(self, inputs: list[str], page_uris: list[str]) -> list[str]: - expanded: list[str] = [] - for input_file in inputs: + _sections: dict[str, dict[str, str]] + + def _expand_inputs(self, inputs: list[str | dict[str, str]], page_uris: list[str]) -> dict[str, str]: + expanded: dict[str, str] = {} + for input_item in inputs: + if isinstance(input_item, dict): + input_file, description = next(iter(input_item.items())) + else: + input_file = input_item + description = "" if "*" in input_file: - expanded.extend(fnmatch.filter(page_uris, input_file)) + for match in fnmatch.filter(page_uris, input_file): + expanded[match] = description else: - expanded.append(input_file) + expanded[input_file] = description return expanded def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: @@ -81,6 +90,7 @@ def on_config(self, config: MkDocsConfig) -> MkDocsConfig | None: if config.site_url is None: raise ValueError("'site_url' must be set in the MkDocs configuration to be used with the 'llmstxt' plugin") self.mkdocs_config = config + # A `defaultdict` could be used, but we need to retain the same order between `config.sections` and `md_pages` # (which wouldn't be guaranteed when filling `md_pages` in `on_page_content()`). self.md_pages = {section: [] for section in self.config.sections} @@ -100,10 +110,10 @@ def on_files(self, files: Files, *, config: MkDocsConfig) -> Files | None: # no Modified collection or none. """ page_uris = list(files.src_uris) - - for section_name, file_list in list(self.config.sections.items()): - self.config.sections[section_name] = self._expand_inputs(file_list, page_uris=page_uris) - + self._sections = { + section_name: self._expand_inputs(file_list, page_uris=page_uris) # type: ignore[arg-type] + for section_name, file_list in self.config.sections.items() + } return files def on_page_content(self, html: str, *, page: Page, **kwargs: Any) -> str | None: # noqa: ARG002 @@ -115,8 +125,9 @@ def on_page_content(self, html: str, *, page: Page, **kwargs: Any) -> str | None html: The rendered HTML. page: The page object. """ - for section_name, file_list in self.config.sections.items(): - if page.file.src_uri in file_list: + src_uri = page.file.src_uri + for section_name, files in self._sections.items(): + if src_uri in files: path_md = Path(page.file.abs_dest_path).with_suffix(".md") page_md = _generate_page_markdown( html, @@ -138,10 +149,11 @@ def on_page_content(self, html: str, *, page: Page, **kwargs: Any) -> str | None self.md_pages[section_name].append( _MDPageInfo( - title=page.title if page.title is not None else page.file.src_uri, + title=page.title if page.title is not None else src_uri, path_md=path_md, md_url=md_url, content=page_md, + description=files[src_uri], ), ) @@ -169,10 +181,10 @@ def on_post_build(self, *, config: MkDocsConfig, **kwargs: Any) -> None: # noqa for section_name, file_list in self.md_pages.items(): markdown += f"## {section_name}\n\n" - for page_title, path_md, md_url, content in file_list: + for page_title, path_md, md_url, content, desc in file_list: path_md.write_text(content, encoding="utf8") _logger.debug(f"Generated MD file to {path_md}") - markdown += f"- [{page_title}]({md_url})\n" + markdown += f"- [{page_title}]({md_url}){(': ' + desc) if desc else ''}\n" markdown += "\n" output_file.write_text(markdown, encoding="utf8") diff --git a/tests/conftest.py b/tests/conftest.py index 3be27ba..517fe7f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1 +1,48 @@ """Configuration for the pytest test suite.""" + +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest +from mkdocs.config.defaults import MkDocsConfig + +if TYPE_CHECKING: + from mkdocs_llmstxt._internal.plugin import MkdocsLLMsTxtPlugin + + +@pytest.fixture(name="mkdocs_conf") +def fixture_mkdocs_conf(request: pytest.FixtureRequest, tmp_path: Path) -> MkDocsConfig: + """Yield a MkDocs configuration object.""" + while hasattr(request, "_parent_request") and hasattr(request._parent_request, "_parent_request"): + request = request._parent_request + params = getattr(request, "param", {}) + config = params.get("config", {}) + pages = params.get("pages", {}) + conf = MkDocsConfig() + conf.load_dict( + { + "site_name": "Test Project", + "site_url": "https://example.org/", + "site_dir": str(tmp_path / "site"), + "docs_dir": str(tmp_path / "docs"), + **config, + }, + ) + Path(conf.docs_dir).mkdir(exist_ok=True) + for page, content in pages.items(): + page_file = Path(conf.docs_dir, page) + page_file.parent.mkdir(exist_ok=True) + page_file.write_text(content) + assert conf.validate() == ([], []) + if "toc" not in conf.markdown_extensions: + # Guaranteed to be added by MkDocs. + conf.markdown_extensions.insert(0, "toc") + return conf + + +@pytest.fixture(name="plugin") +def fixture_plugin(mkdocs_conf: MkDocsConfig) -> MkdocsLLMsTxtPlugin: + """Return a plugin instance.""" + return mkdocs_conf.plugins["llmstxt"] # type: ignore[return-value] diff --git a/tests/test_plugin.py b/tests/test_plugin.py index b892496..ad85dab 100644 --- a/tests/test_plugin.py +++ b/tests/test_plugin.py @@ -1,11 +1,57 @@ """Tests for the plugin.""" +from pathlib import Path + import pytest -from duty.tools import mkdocs +from mkdocs.commands.build import build +from mkdocs.config.defaults import MkDocsConfig + + +@pytest.mark.parametrize( + "mkdocs_conf", + [ + { + "config": { + "plugins": [ + { + "llmstxt": { + "full_output": "llms-full.txt", + "sections": { + "Index": ["index.md"], + "Usage": [{"page1.md": "Some usage docs."}], + }, + }, + }, + ], + }, + "pages": { + "index.md": "# Hello world", + "page1.md": "# Usage\n\nSome paragraph.", + }, + }, + ], + indirect=["mkdocs_conf"], +) +def test_plugin(mkdocs_conf: MkDocsConfig) -> None: + """Test that page descriptions are correctly handled and included in output.""" + build(config=mkdocs_conf) + + llmstxt = Path(mkdocs_conf.site_dir, "llms.txt") + assert llmstxt.exists() + llmstxt_content = llmstxt.read_text() + assert "Some usage docs." in llmstxt_content + assert "Some paragraph." not in llmstxt_content + + llmsfulltxt = Path(mkdocs_conf.site_dir, "llms-full.txt") + assert llmsfulltxt.exists() + llmsfulltxt_content = llmsfulltxt.read_text() + assert "Some usage docs." not in llmsfulltxt_content + assert "Some paragraph." in llmsfulltxt_content + indexmd = Path(mkdocs_conf.site_dir, "index.md") + assert indexmd.exists() + assert "Hello world" in indexmd.read_text() -def test_plugin() -> None: - """Run the plugin.""" - with pytest.raises(expected_exception=SystemExit) as exc: - mkdocs.build()() - assert exc.value.code == 0 + page1md = Path(mkdocs_conf.site_dir, "page1/index.md") + assert page1md.exists() + assert "Some paragraph." in page1md.read_text()