diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 0000000..d56abbf --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Set the default behavior, in case people don't have core.autocrlf set. +* text=auto eol=lf diff --git a/.github/ISSUE_TEMPLATE/bug-report.yaml b/.github/ISSUE_TEMPLATE/bug-report.yaml new file mode 100644 index 0000000..7877f8f --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug-report.yaml @@ -0,0 +1,41 @@ +name: "Bug Report" +description: "Report software deficiencies" +labels: ["bug"] +body: + - type: "markdown" + attributes: + value: | + Use this form to report any functional or performance bugs you've found in the software. + + Be sure to check if your [issue](https://github.com/y-scope/zstd-ffi-js/issues) has already been + reported. + + - type: "textarea" + attributes: + label: "Bug" + description: "Describe what's wrong and if applicable, what you expected instead." + validations: + required: true + + - type: "input" + attributes: + label: "Spider version" + description: "The release version number or development commit hash that has the bug." + placeholder: "Version number or commit hash" + validations: + required: true + + - type: "textarea" + attributes: + label: "Environment" + description: "The environment in which you're running Spider." + placeholder: "OS version, Docker version, etc." + validations: + required: true + + - type: "textarea" + attributes: + label: "Reproduction steps" + description: "List each step required to reproduce the bug." + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/config.yaml b/.github/ISSUE_TEMPLATE/config.yaml new file mode 100644 index 0000000..0086358 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yaml @@ -0,0 +1 @@ +blank_issues_enabled: true diff --git a/.github/ISSUE_TEMPLATE/feature-request.yaml b/.github/ISSUE_TEMPLATE/feature-request.yaml new file mode 100644 index 0000000..50e203c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature-request.yaml @@ -0,0 +1,22 @@ +name: "Feature/Change Request" +description: "Request a feature or change" +labels: ["enhancement"] +body: + - type: "markdown" + attributes: + value: | + Use this form to request a feature/change in the software, or the project as a whole. + + - type: "textarea" + attributes: + label: "Request" + description: "Describe your request and why it's important." + validations: + required: true + + - type: "textarea" + attributes: + label: "Possible implementation" + description: "Describe any implementations you have in mind." + validations: + required: false diff --git a/README.md b/README.md new file mode 100644 index 0000000..c285cbf --- /dev/null +++ b/README.md @@ -0,0 +1,17 @@ +zstd-ffi-js is a JavaScript library that provides bindings to the Zstandard compression library. + +# Docs + +You can find our docs [online][zstd-ffi-js-docs]. + +## Contributing + +We welcome contributions! Please see our [contributing guidelines][contributing-guidelines] for more +information. + +## License + +This project is licensed under the Apache License 2.0. See the [LICENSE](LICENSE) file for details. + +[contributing-guidelines]: https://docs.yscope.com/zstd-ffi-js/main/dev-docs/contributing +[zstd-ffi-js-docs]: https://docs.yscope.com/zstd-ffi-js/main/ diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..46a5f5e --- /dev/null +++ b/docs/README.md @@ -0,0 +1,42 @@ +# Docs + +This directory contains the files necessary to generate a Sphinx-based documentation website for +this project: + +* `conf` - Configuration files +* `src` - The actual docs + +## Requirements + +* [Node.js] >= 16 to be able to [view the output](#viewing-the-output) +* Python 3.10 or higher +* [Task] 3.40.0 or higher + +## Build commands + +* Build the site incrementally: + + ```shell + task docs:site + ``` + + * The output of the build will be in `../build/docs/html`. + +* Clean up the build: + + ```shell + task docs:clean + ``` + +## Viewing the output + +```shell +task docs:serve +``` + +The command above will install [http-server] and serve the built docs site; `http-server` will print +the address it binds to (usually http://localhost:8080). + +[http-server]: https://www.npmjs.com/package/http-server +[Node.js]: https://nodejs.org/en/download/current +[Task]: https://taskfile.dev/ \ No newline at end of file diff --git a/docs/conf/conf.py b/docs/conf/conf.py new file mode 100644 index 0000000..51c3a7b --- /dev/null +++ b/docs/conf/conf.py @@ -0,0 +1,77 @@ +# -- Project information ------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "zstd-ffi-js" + +# NOTE: We don't include a period after "Inc" since the theme adds one already. +copyright = "2025 YScope Inc" + +# -- General configuration ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "myst_parser", + "sphinx_copybutton", + "sphinx_design", + "sphinx.ext.autodoc", + "sphinx.ext.viewcode", +] + +# -- MyST extensions ----------------------------------------------------------- +# https://myst-parser.readthedocs.io/en/stable/syntax/optional.html +myst_enable_extensions = [ + "attrs_block", + "colon_fence", +] + +myst_heading_anchors = 4 + +# -- Sphinx autodoc options ---------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration + +autoclass_content = "class" +autodoc_class_signature = "separated" +autodoc_typehints = "description" + +# -- HTML output options ------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_favicon = "https://docs.yscope.com/_static/favicon.ico" +html_title = project +html_show_copyright = True + +html_static_path = ["../src/_static"] + +html_theme = "pydata_sphinx_theme" + +# -- Theme options ------------------------------------------------------------- +# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/layout.html + +html_theme_options = { + "footer_start": ["copyright"], + "footer_center": [], + "footer_end": ["theme-version"], + "navbar_start": ["navbar-logo"], + "navbar_end": ["navbar-icon-links", "theme-switcher"], + "primary_sidebar_end": [], + "secondary_sidebar_items": ["page-toc", "edit-this-page"], + "show_prev_next": False, + "use_edit_page_button": True, +} + +# -- Theme source buttons ------------------------------------------------------ +# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/source-buttons.html + +html_context = { + "github_user": "y-scope", + "github_repo": "zstd-ffi-js", + "github_version": "main", + "doc_path": "docs/src", +} + +# -- Theme custom CSS and JS --------------------------------------------------- +# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/static_assets.html + + +def setup(app): + app.add_css_file("custom.css") diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 0000000..843d45b --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,5 @@ +myst-parser>=4.0.0 +pydata-sphinx-theme>=0.16.1 +sphinx_design>=0.6.1 +sphinx-copybutton>=0.5.2 +sphinx>=8.1.3 diff --git a/docs/src/_static/custom.css b/docs/src/_static/custom.css new file mode 100644 index 0000000..f25a669 --- /dev/null +++ b/docs/src/_static/custom.css @@ -0,0 +1,32 @@ +html[data-theme="dark"], html[data-theme="light"] { + --pst-color-primary: #3399ff; + --pst-color-secondary: #9580ff; +} + +a { + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +/* +Use the bottom border that's used for indicating the current page as the hover style (for a more +cohesive look). +NOTE: This selector matches the one in pydata-sphinx-theme +pydata/pydata-sphinx-theme@v0.14.4/src/pydata_sphinx_theme/assets/styles/sections/_header.scss#L86 +*/ +.bd-header .navbar-nav li a.nav-link:hover { + border-bottom: max(3px,.1875rem,.12em) solid var(--pst-color-secondary); + text-decoration: none; +} + +/* +Remove margin from sidebar-primary-items__end so that we don't have an unnecessary scrollbar. We're +not using the end items currently. +*/ +.bd-sidebar-primary .sidebar-primary-items__end { + margin-top: 0; + margin-bottom: 0; +} diff --git a/docs/src/dev-docs/building-zstd-wasm.md b/docs/src/dev-docs/building-project.md similarity index 87% rename from docs/src/dev-docs/building-zstd-wasm.md rename to docs/src/dev-docs/building-project.md index ae15427..69f530a 100644 --- a/docs/src/dev-docs/building-zstd-wasm.md +++ b/docs/src/dev-docs/building-project.md @@ -1,6 +1,6 @@ -# Building `zstd-wasm` +# Building the project -This document explains how to build the Zstandard WASM library `zstd-wasm` from source. +This document explains how to build the project. ## Requirements * CMake 3.16 or higher diff --git a/docs/src/dev-docs/contributing.md b/docs/src/dev-docs/contributing.md new file mode 100644 index 0000000..e7e2dc6 --- /dev/null +++ b/docs/src/dev-docs/contributing.md @@ -0,0 +1,9 @@ +# Contributing + +Thank you for your interest in contributing to zstd-ffi-js! This document provides guidelines for +contributing to the project. + +:::{note} +This documentation is still under development. Please check back later for complete contribution +guidelines. +::: \ No newline at end of file diff --git a/docs/src/dev-docs/index.md b/docs/src/dev-docs/index.md new file mode 100644 index 0000000..19dc417 --- /dev/null +++ b/docs/src/dev-docs/index.md @@ -0,0 +1,38 @@ +# Developer docs + +This section contains docs for developing zstd-ffi-js. Choose one of the sections below +or use the left sidebar (if it's hidden, click the icon) to navigate to +specific docs. + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: building-project +Building the project +^^^ +How to build zstd-ffi-js. +::: + +:::{grid-item-card} +:link: contributing +Contributing +^^^ +How to contribute to zstd-ffi-js. +::: + +:::{grid-item-card} +:link: testing +Testing +^^^ +How to test zstd-ffi-js. +::: +:::: + +:::{toctree} +:hidden: + +building-project.md +contributing.md +testing.md +::: \ No newline at end of file diff --git a/docs/src/dev-docs/testing.md b/docs/src/dev-docs/testing.md new file mode 100644 index 0000000..75eb62a --- /dev/null +++ b/docs/src/dev-docs/testing.md @@ -0,0 +1,7 @@ +# Testing + +This document explains how to test zstd-ffi-js. + +:::{note} +This documentation is still under development. Please check back later for complete testing instructions. +::: \ No newline at end of file diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 0000000..4366358 --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,30 @@ +# zstd-ffi-js + +zstd-ffi-js is a JavaScript library that provides bindings to the Zstandard compression library through FFI (Foreign Function Interface). + +This documentation is organized into the following sections: + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: user-docs/index +🧑 User docs +^^^ +Documentation for users who want to use zstd-ffi-js in their projects. +::: + +:::{grid-item-card} +:link: dev-docs/index +🛠 Developer docs +^^^ +Documentation for developers who want to contribute to zstd-ffi-js. +::: +:::: + +:::{toctree} +:hidden: + +user-docs/index.md +dev-docs/index.md +::: \ No newline at end of file diff --git a/docs/src/user-docs/guides-overview.md b/docs/src/user-docs/guides-overview.md new file mode 100644 index 0000000..0a1efe9 --- /dev/null +++ b/docs/src/user-docs/guides-overview.md @@ -0,0 +1,14 @@ +# Overview + +The tutorials below guide you on how to use and operate zstd-ffi-js. + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: quick-start +Quick start +^^^ +How to get started with using zstd-ffi-js in your project. +::: +:::: \ No newline at end of file diff --git a/docs/src/user-docs/index.md b/docs/src/user-docs/index.md new file mode 100644 index 0000000..a322973 --- /dev/null +++ b/docs/src/user-docs/index.md @@ -0,0 +1,24 @@ +# User docs + +This section contains docs for using and operating zstd-ffi-js. Choose one of the sections below or use +the left sidebar (if it's hidden, click the icon) to navigate to specific +docs. + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: guides-overview +Guides +^^^ +Guides for using and operating zstd-ffi-js. +::: +:::: + +:::{toctree} +:hidden: +:caption: Guides + +guides-overview.md +quick-start.md +::: \ No newline at end of file diff --git a/docs/src/user-docs/quick-start.md b/docs/src/user-docs/quick-start.md new file mode 100644 index 0000000..c81ef8e --- /dev/null +++ b/docs/src/user-docs/quick-start.md @@ -0,0 +1,7 @@ +# Quick start + +This guide explains how to get started with using zstd-ffi-js in your project. + +:::{note} +This documentation is still under development. Please check back later for complete instructions. +::: \ No newline at end of file diff --git a/package.json b/package.json index 6b5b8f1..e8ae5f2 100644 --- a/package.json +++ b/package.json @@ -3,6 +3,11 @@ "version": "0.0.1", "description": "A TypeScript/JavaScript Zstandard library with decompression support and planned compression support", "type": "module", + "scripts": { + "docs": "npm run docs:serve", + "docs:build": "task docs:site", + "docs:serve": "task docs:serve" + }, "author": "YScope Inc. ", "license": "Apache-2.0", "homepage": "https://github.com/y-scope/zstd-ffi-js#readme", diff --git a/Taskfile.yml b/taskfile.yaml similarity index 96% rename from Taskfile.yml rename to taskfile.yaml index c9aee4c..13cedb5 100644 --- a/Taskfile.yml +++ b/taskfile.yaml @@ -2,6 +2,7 @@ version: "3" includes: deps: "taskfiles/deps.yaml" + docs: "taskfiles/docs.yaml" utils: "tools/yscope-dev-utils/exports/taskfiles/utils/utils.yaml" vars: @@ -86,4 +87,5 @@ tasks: internal: true silent: true run: "once" - cmds: ["mkdir -p '{{.G_BUILD_DIR}}'"] + cmds: + - "mkdir -p '{{.G_BUILD_DIR}}'" diff --git a/taskfiles/docs.yaml b/taskfiles/docs.yaml new file mode 100644 index 0000000..0f10d6f --- /dev/null +++ b/taskfiles/docs.yaml @@ -0,0 +1,116 @@ +version: "3" + +vars: + # Paths + G_DOCS_BUILD_DIR: "{{.G_BUILD_DIR}}/docs/html" + G_DOCS_VENV_DIR: "{{.G_BUILD_DIR}}/docs-venv" + G_NODE_DEPS_DIR: "{{.G_BUILD_DIR}}/docs-node" + + # Target checksum files + G_DOCS_VENV_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/docs#docs-venv.md5" + +tasks: + clean: + cmds: + - "rm -rf '{{.G_DOCS_BUILD_DIR}}'" + + serve: + deps: + - "http-server" + - "site" + cmds: + - "npm --prefix '{{.G_NODE_DEPS_DIR}}' exec http-server '{{.G_DOCS_BUILD_DIR}}' -c-1" + + site: + vars: + CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK | replace \":\" \"#\"}}.md5" + OUTPUT_DIR: "{{.G_DOCS_BUILD_DIR}}" + dir: "{{.ROOT_DIR}}/docs" + deps: + - ":init" + - task: ":utils:checksum:validate" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + - "docs-venv" + cmds: + # Call `clean` before building since `sphinx-build --write-all --fresh-env` isn't always + # equivalent to building from scratch. + - task: "clean" + - "python3 '{{.ROOT_DIR}}/tools/scripts/find-broken-docs-links.py'" + - |- + . "{{.G_DOCS_VENV_DIR}}/bin/activate" + sphinx-build \ + --write-all \ + --fresh-env \ + --conf-dir conf \ + --nitpicky \ + --fail-on-warning \ + --keep-going \ + --builder html \ + src "{{.OUTPUT_DIR}}" + # This command must be last + - task: ":utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + sources: + - "{{.G_DOCS_VENV_CHECKSUM_FILE}}" + - "{{.ROOT_DIR}}/taskfile.yaml" + - "{{.TASKFILE}}" + - "conf/**/*" + - "src/**/*" + generates: ["{{.CHECKSUM_FILE}}"] + + docs-venv: + internal: true + vars: + CHECKSUM_FILE: "{{.G_DOCS_VENV_CHECKSUM_FILE}}" + OUTPUT_DIR: "{{.G_DOCS_VENV_DIR}}" + REQUIREMENTS_FILE: "docs/requirements.txt" + deps: + - ":init" + - task: ":utils:checksum:validate" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + cmds: + - task: ":utils:misc:create-venv" + vars: + LABEL: "docs" + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + REQUIREMENTS_FILE: "{{.REQUIREMENTS_FILE}}" + # This command must be last + - task: ":utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + sources: + - "{{.REQUIREMENTS_FILE}}" + - "{{.ROOT_DIR}}/taskfile.yaml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + + http-server: + internal: true + vars: + CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK | replace \":\" \"#\"}}.md5" + OUTPUT_DIR: "{{.G_NODE_DEPS_DIR}}" + deps: + - ":init" + - task: ":utils:checksum:validate" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + cmds: + - "rm -rf '{{.OUTPUT_DIR}}'" + - "npm --prefix '{{.OUTPUT_DIR}}' install http-server" + # This command must be last + - task: ":utils:checksum:compute" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + INCLUDE_PATTERNS: ["{{.OUTPUT_DIR}}"] + sources: + - "{{.ROOT_DIR}}/taskfile.yaml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] diff --git a/tools/scripts/find-broken-docs-links.py b/tools/scripts/find-broken-docs-links.py new file mode 100644 index 0000000..1a1d168 --- /dev/null +++ b/tools/scripts/find-broken-docs-links.py @@ -0,0 +1,110 @@ +import os +import subprocess +import sys +from pathlib import Path +from typing import List + + +def main(argv: List[str] = None): + if argv is None: + argv = sys.argv + + repo_root = _get_repo_root() + + found_violation = False + + # Check for docs.yscope.com links with ".md" suffixes + if _check_tracked_files( + r"docs\.yscope\.com/.+\.md", + repo_root, + repo_root, + 'docs.yscope.com links cannot have ".md" suffixes.', + ): + found_violation = True + + # Check for sphinx :link: attributes that have ".md" suffixes + if _check_tracked_files( + r":link:[[:space:]]*.+\.md", + repo_root, + repo_root / "docs", + 'sphinx :link: attributes cannot have ".md" suffixes', + ): + found_violation = True + + if found_violation: + return 1 + + return 0 + + +def _get_repo_root() -> Path: + path_str = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], cwd=Path(__file__).parent, text=True + ) + return Path(path_str.strip()) + + +def _check_tracked_files( + pattern: str, repo_root: Path, dir_to_search: Path, error_msg: str +) -> bool: + """ + Check for a pattern in all tracked files in the repo (except this script). + :param pattern: The pattern to search for. + :param repo_root: The root of the repository. + :param dir_to_search: The directory to search in. + :param error_msg: Error message if the pattern is found. + :return: Whether the pattern was found in any file. + """ + found_matches = False + + # NOTE: "-z" ensures the paths won't be quoted (while delimiting them using '\0') + for path_str in subprocess.check_output( + [ + "git", + "ls-files", + "--cached", + "--exclude-standard", + "-z", + str(dir_to_search.relative_to(repo_root)), + ], + cwd=repo_root, + text=True, + ).split("\0"): + path = Path(path_str) + + # Skip directories and this script + if path == __file__ or (repo_root / path).is_dir(): + continue + + try: + for match in subprocess.check_output( + ["grep", "--extended-regexp", "--line-number", "--with-filename", pattern, path], + cwd=repo_root, + text=True, + ).splitlines(): + _parse_and_print_match(match, error_msg) + found_matches = True + except subprocess.CalledProcessError as ex: + if ex.returncode != 1: + print(f"Failed to grep '{path}' - exit status {ex.returncode}.", file=sys.stderr) + + return found_matches + + +def _parse_and_print_match(match: str, error_msg: str): + """ + Parses and prints grep matches in a format relevant to the current environment. + :param match: The match to parse and print. + :param error_msg: Error message if the pattern is found. + """ + if os.getenv("GITHUB_ACTIONS") == "true": + # Print a GitHub Actions error annotation + file, line, _ = match.split(":", 2) + print(f"::error file={file},line={line}::{error_msg}") + else: + print(error_msg, file=sys.stderr) + print(match, file=sys.stderr) + + +if "__main__" == __name__: + sys.exit(main(sys.argv))