Merge branch 'develop' into feature/dul-extensions

Author: mdbenito

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.9.3.dev0
+current_version = 0.10.1.dev0
 commit = False
 tag = False
 allow_dirty = False

.gitattributes

@@ -2,3 +2,4 @@ notebooks/*.ipynb -linguist-detectable
 *.png filter=lfs diff=lfs merge=lfs -text
 *.svg filter=lfs diff=lfs merge=lfs -text
 logo.svg -filter=lfs -diff=lfs -merge=lfs text
+*.pkl filter=lfs diff=lfs merge=lfs -text

.github/workflows/main.yaml

@@ -112,13 +112,13 @@ jobs:
       group_number: ${{ matrix.group_number }}
     needs: [ code-quality ]
 
-  push-docs-and-release-testpypi:
-    name: Publish documentation # and maybe release package to TestPyPI
+  publish-docs:
+    name: Publish documentation
     runs-on: ubuntu-22.04
     needs: [docs, group-tests, notebook-tests]
-    if: ${{ github.ref == 'refs/heads/develop' }}
+    if: ${{ github.ref == 'refs/heads/develop' && github.event_name != 'pull_request'  }}
     concurrency:
-      group: publish
+      group: publish-docs
     steps:
       - uses: actions/checkout@v4
         with:
@@ -140,14 +140,37 @@ jobs:
           title: Development
           email: ${{ env.GITHUB_BOT_EMAIL }}
           username: ${{ env.GITHUB_BOT_USERNAME }}
-      # Deactivated until we solve the issue with the credentials (Jan. 2025)
-      # - name: Build and publish to TestPyPI
-      #   env:
-      #     TWINE_USERNAME: __token__
-      #     TWINE_PASSWORD: ${{ secrets.TEST_PYPI_PASSWORD }}
-      #   run: |
-      #     set -x
-      #     export BUILD_NUMBER=$GITHUB_RUN_NUMBER
-      #     bump2version --no-tag --no-commit --verbose --serialize '{major}.{minor}.{patch}.{release}{$BUILD_NUMBER}' boguspart
-      #     python setup.py sdist bdist_wheel
-      #     twine upload -r testpypi --verbose --non-interactive dist/*
+
+
+  publish-testpypi:
+    name: Publish package to TestPyPI
+    runs-on: ubuntu-22.04
+    needs: [group-tests, notebook-tests]
+    if: ${{ github.ref == 'refs/heads/develop' && github.event_name != 'pull_request' }}
+    concurrency:
+      group: publish-testpypi
+    permissions:
+      id-token: write
+    environment:
+      name: testpypi
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          lfs: true
+      - name: Setup Python 3.9
+        uses: ./.github/actions/python
+        with:
+          python_version: 3.9
+      - name: Bump version and build package
+        run: |
+          set -x
+          export BUILD_NUMBER=$GITHUB_RUN_NUMBER
+          bump2version --no-tag --no-commit --verbose --serialize '{major}.{minor}.{patch}.{release}{$BUILD_NUMBER}' boguspart
+          python setup.py sdist bdist_wheel
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          print-hash: true
+          verbose: true

.github/workflows/publish.yaml

@@ -23,8 +23,13 @@ env:
 jobs:
   publish:
     runs-on: ubuntu-22.04
+    environment:
+      name: publish
+      url: https://pypi.org/pydvl
     concurrency:
       group: publish
+    permissions:
+      id-token: write
     steps:
       - name: Checking out last commit in release
         if: ${{ github.event_name != 'workflow_dispatch' }}
@@ -54,6 +59,14 @@ jobs:
           # Make the version available as env variable for next steps
           echo CURRENT_VERSION=$CURRENT_VERSION >> $GITHUB_ENV
         shell: bash
+      - name: Build dist
+        run: |
+          python setup.py sdist bdist_wheel
+      - name: Publish to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          verbose: true
+          print-hash: true
       - name: Deploy Docs
         uses: ./.github/actions/deploy-docs
         with:
@@ -63,10 +76,3 @@ jobs:
           email: ${{ env.GITHUB_BOT_EMAIL }}
           username: ${{ env.GITHUB_BOT_USERNAME }}
           set-default: 'true'
-      - name: Build and publish to PyPI
-        env:
-          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
-          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
-        run: |
-          python setup.py sdist bdist_wheel
-          twine upload --verbose --non-interactive dist/*

.github/workflows/run-notebook-tests-workflow.yaml

@@ -26,11 +26,12 @@ jobs:
     - name: Free Disk Space (Ubuntu)
       uses: jlumbroso/free-disk-space@main
       with:
-        large-packages: false
+        large-packages: true
         docker-images: false
     - uses: actions/checkout@v4
       with:
         fetch-depth: 0
+        lfs: true
     - name: Setup Python ${{ inputs.python_version }}
       uses: ./.github/actions/python
       with:

.gitignore

@@ -110,7 +110,7 @@ celerybeat.pid
 .venv
 env/
 venv/
-venv38/
+venv39/
 ENV/
 env.bak/
 venv.bak/
@@ -141,9 +141,18 @@ pylint.html
 runs/
 data/models/
 *.pkl
+!data/*.pkl
 
 # Docs
 docs_build
 
 # pytest-profiling
 prof/
+
+# JS tooling
+node_modules/
+package.json
+package-lock.json
+
+#
+.serena

.test_durations

@@ -1521,8 +1521,8 @@
     "tests/valuation/methods/test_semivalues.py::test_coefficients[BetaShapleyValuation-kwargs1-10]": 0.0016590010000072652,
     "tests/valuation/methods/test_semivalues.py::test_coefficients[BetaShapleyValuation-kwargs2-100]": 0.0022294990000091275,
     "tests/valuation/methods/test_semivalues.py::test_coefficients[BetaShapleyValuation-kwargs2-10]": 0.003863207999984297,
-    "tests/valuation/methods/test_semivalues.py::test_coefficients[DataBanzhafValuation-kwargs3-100]": 0.001800666000065121,
-    "tests/valuation/methods/test_semivalues.py::test_coefficients[DataBanzhafValuation-kwargs3-10]": 0.0016530420000435697,
+    "tests/valuation/methods/test_semivalues.py::test_coefficients[BanzhafValuation-kwargs3-100]": 0.001800666000065121,
+    "tests/valuation/methods/test_semivalues.py::test_coefficients[BanzhafValuation-kwargs3-10]": 0.0016530420000435697,
     "tests/valuation/methods/test_semivalues.py::test_coefficients[ShapleyValuation-kwargs4-100]": 0.0018769589999578784,
     "tests/valuation/methods/test_semivalues.py::test_coefficients[ShapleyValuation-kwargs4-10]": 0.0016063749999375432,
     "tests/valuation/methods/test_semivalues.py::test_msr_banzhaf[5]": 9.342398666999998,
@@ -1636,10 +1636,10 @@
     "tests/valuation/scorers/test_classwise.py::test_classwise_scorer[test_data2-expected_scores2]": 0.0025690839999974457,
     "tests/valuation/scorers/test_scorers.py::test_compose_score": 0.0019082069999996065,
     "tests/valuation/scorers/test_scorers.py::test_scorer": 0.001976999999998341,
-    "tests/valuation/test_interface.py::test_data_banzhaf_valuation[1]": 0.0836418330000015,
-    "tests/valuation/test_interface.py::test_data_banzhaf_valuation[2]": 1.2780167490000025,
-    "tests/valuation/test_interface.py::test_data_beta_shapley_valuation[1]": 4.139234666999997,
-    "tests/valuation/test_interface.py::test_data_beta_shapley_valuation[2]": 3.603092916999998,
+    "tests/valuation/test_interface.py::test_banzhaf_valuation[1]": 0.0836418330000015,
+    "tests/valuation/test_interface.py::test_banzhaf_valuation[2]": 1.2780167490000025,
+    "tests/valuation/test_interface.py::test_beta_shapley_valuation[1]": 4.139234666999997,
+    "tests/valuation/test_interface.py::test_beta_shapley_valuation[2]": 3.603092916999998,
     "tests/valuation/test_interface.py::test_shapley_valuation[1]": 0.27120083299999465,
     "tests/valuation/test_interface.py::test_shapley_valuation[2]": 0.15037520699999618,
     "tests/valuation/test_interface.py::test_data_utility_learning[1]": 0.026216332999993597,
@@ -1781,10 +1781,6 @@
     "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[owen-kwargs1-scorer0-0.2-2-0-21]": 6.573138832000012,
     "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[owen_antithetic-kwargs2-scorer0-0.2-2-0-21]": 10.124256999999972,
     "tests/value/shapley/test_montecarlo.py::test_linear_montecarlo_with_outlier[permutation_montecarlo-kwargs0-scorer0-0.2-2-0-21]": 2.7115268339999545,
-    "tests/value/shapley/test_montecarlo.py::test_montecarlo_shapley_housing_dataset[12-3-12-combinatorial_montecarlo-kwargs0]": 0.16786966001382098,
-    "tests/value/shapley/test_montecarlo.py::test_montecarlo_shapley_housing_dataset[12-3-12-owen-kwargs1]": 17.011920137971174,
-    "tests/value/shapley/test_montecarlo.py::test_montecarlo_shapley_housing_dataset[12-3-12-owen_antithetic-kwargs2]": 35.88025256394758,
-    "tests/value/shapley/test_montecarlo.py::test_montecarlo_shapley_housing_dataset[12-3-4-group_testing-kwargs3]": 0.25901710899779573,
     "tests/value/shapley/test_montecarlo.py::test_seed[combinatorial_montecarlo-kwargs0-test_game0]": 0.04085670800000685,
     "tests/value/shapley/test_montecarlo.py::test_seed[group_testing-kwargs3-test_game0]": 0.23488145900003587,
     "tests/value/shapley/test_montecarlo.py::test_seed[owen-kwargs1-test_game0]": 0.30296191700003305,

CHANGELOG.md

@@ -2,9 +2,38 @@
 
 ## Unreleased
 
+### Added
+
+- Support for `torch.Tensor` as underlying data type in `Dataset` and
+  `GroupedDataset`
+  [PR #673](https://github.com/aai-institute/pyDVL/pull/673)
+- Support for pytorch models in most valuation methods when wrapped in
+  classes implementing the protocol `TorchSupervisedModel`, e.g. by using
+  [skorch.NeuralNetClassifier](https://skorch.readthedocs.io/en/stable/classifier.html)
+  models
+  [PR #673](https://github.com/aai-institute/pyDVL/pull/673)
+
+### Fixed
+
+- Issues with `Dataset` indexing
+  [PR #673](https://github.com/aai-institute/pyDVL/pull/673)
+
+### Changed
+
+
+## v0.10.0 - 💥📚🐞🆕 New valuation interface, improved docs, new methods, breaking changes and tons of improvements
+
 
 ### Added
 
+- Simple result serialization to resume computation of values
+  [PR #666](https://github.com/aai-institute/pyDVL/pull/666)
+- Simple memory monitor / reporting
+  [PR #663](https://github.com/aai-institute/pyDVL/pull/663)
+- New stopping criterion `MaxSamples`
+  [PR #661](https://github.com/aai-institute/pyDVL/pull/661)
+- New pseudo-stopping criterion `History` to record value updates during fitting
+  [PR #653](https://github.com/aai-institute/pyDVL/pull/653)
 - Introduced `UtilityModel` and two implementations `IndicatorUtilityModel`
   and `DeepSetsUtilityModel` for data utility learning
   [PR #650](https://github.com/aai-institute/pyDVL/pull/650)
@@ -25,6 +54,7 @@
 - Refactor MSR Banzhaf semivalues with the new sampler architecture.
   [PR #605](https://github.com/aai-institute/pyDVL/pull/605)
   [PR #641](https://github.com/aai-institute/pyDVL/pull/641)
+  [PR #653](https://github.com/aai-institute/pyDVL/pull/653)
 - Refactor group-testing shapley values with new sampler architecture
   [PR #602](https://github.com/aai-institute/pyDVL/pull/602)
 - Refactor least-core data valuation methods with more supported sampling
@@ -56,8 +86,10 @@
 
 ### Fixed
 
-- Fixed `show_warnings=False` not being respected in subprocesses
+- Fixed `show_warnings=False` not being respected in subprocesses. Introduced
+  `suppress_warninigs` decorator for more flexibility
   [PR #647](https://github.com/aai-institute/pyDVL/pull/647)
+  [PR #662](https://github.com/aai-institute/pyDVL/pull/662)
 - Fixed several bugs in diverse stopping criteria, including: iteration counts,
   computing completion, resetting, nested composition
   [PR #641](https://github.com/aai-institute/pyDVL/pull/641)
@@ -83,6 +115,13 @@
 
 ### Changed
 
+- Major rewrite of the documentation for the new interface in 0.10.0
+  [PR #653](https://github.com/aai-institute/pyDVL/pull/653)
+- Slicing, comparing and setting of `ValuationResult` behave in a more 
+  natural and consistent way. Several bug fixes.
+  [PR #653](https://github.com/aai-institute/pyDVL/pull/653)
+  [PR #660](https://github.com/aai-institute/pyDVL/pull/660) 
+  [PR #666](https://github.com/aai-institute/pyDVL/pull/666)
 - Switched all semi-value coefficients and sampler weights to log-space in
   order to avoid overflows
   [PR #643](https://github.com/aai-institute/pyDVL/pull/643)
@@ -151,6 +190,8 @@
     - renaming `PreConditioner` -> `Preconditioner`
     - fit to `TensorOperator`
     [PR #601](https://github.com/aai-institute/pyDVL/pull/601)
+  - Bumped `zarr` dependency to v3
+    [PR #668](https://github.com/aai-institute/pyDVL/pull/668) 
 
 
 ## 0.9.2 - 🏗  Bug fixes, logging improvement

CLAUDE.local.md

@@ -15,10 +15,10 @@ If you are interested in setting up a similar project, consider the template
 
 ## Local development
 
-This project uses [black](https://github.com/psf/black) to format code and
+This project uses [ruff](https://github.com/astral-sh/ruff) to lint and format code and
 [pre-commit](https://pre-commit.com/) to invoke it as a git pre-commit hook.
-Consider installing any of [black's IDE
-integrations](https://black.readthedocs.io/en/stable/integrations/editors.html)
+Consider installing any of [ruff's IDE
+integrations](https://docs.astral.sh/ruff/editors/setup/)
 to make your life easier.
 
 Run the following to set up the pre-commit git hook to run before pushes:
@@ -83,7 +83,7 @@ If you use remote execution, don't forget to exclude data paths from deployment
 ## Testing
 
 Automated builds, tests, generation of documentation and publishing are handled
-by [CI pipelines](#CI). Before pushing your changes to the remote we recommend
+by [CI pipelines](#ci). Before pushing your changes to the remote we recommend
 to execute `tox` locally in order to detect mistakes early on and to avoid
 failing pipelines. tox will:
 * run the test suite
@@ -297,6 +297,33 @@ the environment variable `DYLD_FALLBACK_LIBRARY_PATH`:
 export DYLD_FALLBACK_LIBRARY_PATH=$DYLD_FALLBACK_LIBRARY_PATH:/opt/homebrew/lib
 ```
 
+### Automatic API documentation
+
+We use [mkdocstrings](https://mkdocstrings.github.io/) to automatically generate
+API documentation from docstrings, following almost verbatim [this
+recipe](https://mkdocstrings.github.io/recipes/#automatic-code-reference-pages):
+Stubs are generated for all modules on the fly using
+[generate_api_docs.py](https://github.com/aai-institute/pyDVL/blob/develop/build_scripts/generate_api_docs.py) thanks to the pluging
+[mkdocstrings-gen-files](https://github.com/oprypin/mkdocs-gen-files) and
+navigation is generated for
+[mkdocs-literate-nav](https://github.com/oprypin/mkdocs-literate-nav).
+
+With some renaming and using
+[section-index](https://github.com/oprypin/mkdocs-section-index) `__init__.py`
+files are used as entry points for the documentation of a module.
+
+Since very often we re-export symbols in the `__init__.py` files, the automatic
+generation of the documentation skips **all** symbols in those files. If you
+want to document any in particular you can do so by **overriding
+mkdocs_genfiles**: Create a file under `docs/api/pydvl/module/index.md` and add
+your documentation there. For example, to document the whole module and every
+(re-)exported symbol just add this to the file:
+
+```markdown
+::: pydvl.module
+```
+
+
 ### Adding new pages
 
 Navigation is configured in `mkdocs.yaml` using the nav section. We use the
@@ -388,7 +415,7 @@ To add a citation inside a markdown file, use the notation `[@ citekey]` (with
 no space). Alas,
 because of when mkdocs-bibtex enters the pipeline, it won't process docstrings.
 For module documentation, we manually inject html into the markdown files. For
-example, in `pydvl.value.shapley.montecarlo` we have:
+example, in `pydvl.valuation.methods.shapley` we have:
 
 ```markdown
 """
@@ -441,7 +468,7 @@ use braces for legibility like in the first example.
 ### Abbreviations
 
 We keep the abbreviations used in the documentation inside the
-[docs_include/abbreviations.md](https://github.com/aai-institute/pyDVL/blob/develop/docs_includes%2Fabbreviations.md) file.
+[docs_include/abbreviations.md](https://github.com/aai-institute/pyDVL/blob/develop/docs_includes/abbreviations.md) file.
 
 The syntax for abbreviations is:
 
@@ -649,8 +676,18 @@ part):
 ```shell
 build_scripts/release-version.sh 0.1.6
 ```
+This will:
+
+ - Pull latest remote version of develop (fast-forward only) from origin
+ - Create a branch `release/v0.1.6`
+ - Bump version number: 0.1.5.dev1234 ⟶   0.1.6
+ - Merge release branch into master locally and on origin
+ - Tag as `v0.1.6`
+ - Bump version number again to next development pre-release
+ - Merge release branch into develop locally and on origin
+ - Delete release branch
 
-To find out how to use the script, pass the `-h` or `--help` flags:
+For usage details, pass the `-h` or `--help` flags:
 
 ```shell
 build_scripts/release-version.sh --help
@@ -660,11 +697,10 @@ If running in interactive mode (without `-y|--yes`), the script will output a
 summary of pending changes and ask for confirmation before executing the
 actions.
 
-Once this is done, a tag will be created on the repository. You should then
-create a GitHub
+Once the script is done, you should then create a GitHub
 [release](https://docs.github.com/en/repositories/releasing-projects-on-github/managing-releases-in-a-repository#creating-a-release)
-for that tag. That will a trigger a CI pipeline that will automatically create a
-package and publish it from CI to PyPI.
+for the tag that was created. That will a trigger a CI pipeline that will
+automatically create a package and publish it from CI to PyPI.
 
 ### Manual release process