automl
diff --git a/‎CHANGELOG.md
Lines changed: 5 additions & 0 deletions b/‎CHANGELOG.md
Lines changed: 5 additions & 0 deletions
diff --git a/‎CITATION.cff
Lines changed: 1 addition & 1 deletion b/‎CITATION.cff
Lines changed: 1 addition & 1 deletion
diff --git a/‎CONTRIBUTING.md
Lines changed: 4 additions & 6 deletions b/‎CONTRIBUTING.md
Lines changed: 4 additions & 6 deletions
diff --git a/‎MANIFEST.in
Lines changed: 5 additions & 2 deletions b/‎MANIFEST.in
Lines changed: 5 additions & 2 deletions
diff --git a/‎README.md
Lines changed: 17 additions & 103 deletions b/‎README.md
Lines changed: 17 additions & 103 deletions
diff --git a/‎Makefile renamed to ‎carps/build/Makefile
Lines changed: 19 additions & 18 deletions b/‎Makefile renamed to ‎carps/build/Makefile
Lines changed: 19 additions & 18 deletions
diff --git a/‎carps/build/make.py
Lines changed: 1 addition & 1 deletion b/‎carps/build/make.py
Lines changed: 1 addition & 1 deletion
diff --git a/‎container_recipes/benchmarks/BBOB/BBOB.recipe renamed to ‎carps/container/recipes/benchmarks/BBOB/BBOB.recipe
Lines changed: 3 additions & 3 deletions b/‎container_recipes/benchmarks/BBOB/BBOB.recipe renamed to ‎carps/container/recipes/benchmarks/BBOB/BBOB.recipe
Lines changed: 3 additions & 3 deletions
@@ -1,4 +1,9 @@
+# v1.0.1
+- Fix installation via pypi (#185).
+- Update docs (#186).
+
 # v1.0.0
+⚠ Breaking Changes
 Redefined task as an objective function together with an input and output space. Updated configs. Renamed problem to
 objective function and scenario to task type.
 
 
@@ -10,7 +10,7 @@ url: "https://automl.github.io/CARP-S/main/"
 
 repository-code: "https://github.com/automl/CARP-S"
 
-version: "1.0.0"
+version: "1.0.1"
 
 type: "template"
 keywords:
 
@@ -81,19 +81,17 @@ git push origin name-of-your-bugfix-or-feature
 Submit a [pull request](https://github.com/automl/CARP-S/pulls) through the GitHub website!
 
 ## Local Development
-
-### Virtual Environments
-You can try to install all dependencies into one big environment, but probably there are package clashes.
-Therefore, you can build one virtual environment for each optimizer-benchmark combination.
-Either run `scripts/build_envs.sh` to build all existing combinations or copy the combination and run as needed. It will create an environment with name `automlsuite_${OPTIMIZER_CONTAINER_ID}_${BENCHMARK_ID}`.
+To promote compatibility we encourage to enable `numpy>2.0.0` and `ConfigSpace>1.0.0`, and that in general modern
+python versions are supported.
+If there are still package clashes, you can create a virtual env per benchmark and optimizer.
 
 ## Pull Request Guidelines
 Before you submit a pull request, check that it meets these guidelines:
 
 1. The pull request should include tests.
 2. If the pull request adds functionality, the docs should be updated.
 Put your new functionality into a function with a docstring, and add the feature to the list in `README.md`.
-3. The pull request should work for `Python 3.9` and
+3. The pull request should work for `Python 3.9` (ideally newer versions) and
 make sure that the tests pass for all supported Python versions.
 
 ## Testing
 
@@ -4,6 +4,7 @@ include CONTRIBUTING.md
 include LICENSE
 include README.md
 include pyproject.toml
+include carps/build/Makefile
 
 graft carps
 graft docs
@@ -12,14 +13,16 @@ graft tests
 # recursive-include tests *
 recursive-exclude * __pycache__
 recursive-exclude * *.py[co]
-recursive-exclude container_recipes *
 recursive-exclude examples *
 recursive-exclude notebooks *
 recursive-exclude scripts *
 recursive-exclude subselection *
 recursive-exclude lib *
 recursive-exclude env* *
+recursive-exclude carps.egg* *
 recursive-exclude carps/benchmark_data *
+recursive-exclude carps/task_data *
 
-recursive-include docs *.rst Makefile make.bat *.jpg *.png *.gif
+recursive-include docs *.rst make.bat *.jpg *.png *.gif
 recursive-include carps/configs *
+recursive-include carps/container/recipes *
@@ -20,8 +20,6 @@ For more details on CARP-S, please have a look at the
 
 ### Installation from PyPI
 
-⚠️ The installation of the optimizers and benchmarks/tasks currently does not work via pip due to packaging issues of the install scripts. Until this is fixed, please install `carps` from source (see below).
-
 To install CARP-S, you can simply use `pip`:
 
 1. Create virtual env with conda or uv
@@ -76,25 +74,15 @@ $PIP install .
 $PIP install -e .
 ```
 
-If you want to install CARP-S for development, you can use the following command:
+If you want to install CARP-S for development, you can use the following command (from the root of the repo):
 ```bash
-make install-dev
+$PIP install -e .
+python -m carps.build.make install-dev
 ```
 #### Apptainer
-You can also create a container with the env setup by running `apptainer build container/env.sif container/env.def`.
-Then you can execute any carps commands as usual by add this prefix `apptainer exec container/env.sif` before the
-command, e.g. `apptainer exec container/env.sif python -m carps.run +task/... +optimizer/...`.
-There is also an sbatch script to run experiments from the database using the apptainer on a slurm cluster
-(`sbatch scripts/container_run_from_db.sh`). You might need to adapt the array size and the number of repetitions
-according to the number of experiments you can run.
-
-PS.: On some clusters you might need to load the module apptainer like so `module load tools Apptainer`.
-Troubleshooting: If you have problems writing your cache directory, mount-bind it like so
-`apptainer shell --bind $XDG_CACHE_HOME container/env.sif`. This binds the directory `$XDG_CACHE_HOME` in the
-container to the directory `$XDG_CACHE_HOME` on the host.
-If you have problems with `/var/lib/hpobench`, this bind might help: 
-`<hpobench data dir>:/var/lib/hpobench/data`. `<hpobench data dir>` can be found in
-[`.hpobenchrc`](https://github.com/automl/HPOBench/?tab=readme-ov-file#configure-hpobench).
+⚠ This is still experimental.
+You can also use a container as an env, see
+[this guide](https://automl.github.io/CARP-S/latest/installation/#apptainer).
 
 #### A note on python versions
 For python3.12, numpy should be `numpy>=2.0.0`. For python3.10, numpy must be `numpy==1.26.4`, you can simply
@@ -106,13 +94,14 @@ Additionally, you need to install the requirements for the benchmark and optimiz
 
 ⚠ You can specify the directory of the task data by `export CARPS_TASK_DATA_DIR=...`. Please use absolute dirnames.
 The default location is `<carps package location>/task_data`. If you specify a custom dir, always export the env var.
-
+(The carps package location is the root of the package, not of the repo.)
 
 For example, if you want to use the `SMAC3` optimizer and the `BBOB` benchmark, you need to install the
 requirements for both of them via:
 
 ```bash
-# Install options for optimizers and benchmarks (these are Makefile commands, check the Makefile for more commands)
+# Install options for optimizers and benchmarks (these are Makefile commands, check the Makefile at carps/build for
+# more commands)
 # The commands should be separated by a whitespace
 python -m carps.build.make benchmark_bbob optimizer_smac
 ```
@@ -132,7 +121,7 @@ optimizer_smac optimizer_dehb optimizer_nevergrad optimizer_optuna optimizer_ax
 All of the above except `optimizer_hebo` work with python3.12.
 
 You can also install all benchmarks in one go with `benchmarks` and all optimizers with `optimizers`.
-Check the `Makefile` in carps for more details.
+Check the `carps/build/Makefile` in carps for more details.
 
 
 ## Minimal Example
@@ -153,6 +142,7 @@ should be run for all available BBOB tasks (`+task/BBOB=glob(*)`) and for 10 dif
 seed values (seed=range(1,11)).
 
 ## Commands
+For a complete list see the [docs](https://automl.github.io/CARP-S/latest/commands/).
 
 You can run a certain task and optimizer combination directly with Hydra via:
 ```bash
@@ -165,97 +155,21 @@ a file `runcommands_missing.sh` containing the missing runs:
 python -m carps.utils.check_missing <rundir>
 ```
 
-To collect all run data generated by the file logger into csv files, use the following command:
+To collect all run data generated by the file logger into parquet files, use the following command:
 ```bash
 python -m carps.analysis.gather_data <rundir>
 ```
-The csv files are then located in `<rundir>`. `logs.csv` contain the trial info and values and 
-`logs_cfg.csv` contain the experiment  configuration.
+The parquet files are then located in `<rundir>`. `logs.parquet` contain the trial info and values and 
+`logs_cfg.parquet` contain the experiment  configuration.
 The experiments can be matched via the column `experiment_id`.
 
 ## CARPS and MySQL Database
 Per default, `carps` logs to files. This has its caveats: Checking experiment status is a bit more cumbersome (but 
 possible with `python -m carps.utils.check_missing <rundir>` to check for missing/failed experiments) and reading from
 the filesystem takes a long time. For this reason, we can also control and log experiments to a MySQL database with
-`PyExperimenter`.
-
-### Requirements and Configuration
-Requirement: MySQL database is set up.
-
-1. Add a `credentials.yaml` file in `carps/experimenter` with the following content:
-```yaml
-CREDENTIALS:
-  Database:
-      user: someuser
-      password: amazing_password
-  Connection:
-      Standard:
-        server: mysql_server
-        port: 3306 (most likely)
-```
-2. Edit `carps/experimenter/py_experimenter.yaml` by setting:
-```yaml
-PY_EXPERIMENTER:
-  n_jobs: 1
-
-  Database:
-    use_ssh_tunnel: false
-    provider: mysql
-    database: your_database_name
-...
-```
-!!! Note: If you use an ssh tunnel, set `use_ssh_tunnel` to `true` in `carps/experimenter/py_experimenter.yaml`.
-Set up  `carps/experimenter/credentials.yaml` like this:
-```yaml
-CREDENTIALS:
-  Database:
-      user: someuser
-      password: amazing_password
-  Connection:
-      Standard:
-        server: mysql_server
-        port: 3306 (most likely)
-      Ssh:
-        server: 127.0.0.1
-        address: some_host  # hostname as specified in ~/.ssh/config
-        # ssh_private_key_password: null
-        # server: example.sshmysqlserver.com (address from ssh server)
-        # address: example.sslserver.com
-        # port: optional_ssh_port
-        # remote_address: optional_mysql_server_address
-        # remote_port: optional_mysql_server_port
-        # local_address: optional_local_address
-        # local_port: optional_local_port
-        # passphrase: optional_ssh_passphrase
-```
-### Create Experiments
-First, in order for PyExperimenter to be able to pull experiments from the database, we need to fill it.
-The general command looks like this:
-```bash
-python -m carps.experimenter.create_cluster_configs +task=... +optimizer=... -m
-```
-All subset runs were created with `scripts/create_experiments_in_db.sh`.
-
-### Running Experiments
-Now, execute experiments with:
-```bash
-python -m carps.run_from_db 'job_nr_dummy=range(1,1000)' -m
-```
-This will create 1000 multirun jobs, each pulling an experiment from PyExperimenter and executing it.
-
-!!! Note: On most slurm clusters the max array size is 1000.
-!!! Note: On our mysql server location, at most 300 connections at the same time are possible. You can limit your number
-    of parallel jobs with `hydra.launcher.array_parallelism=250`.
-!!! `carps/configs/runfromdb.yaml` configures the run and its resources. Currently defaults for our slurm cluster are
-    configured. If you run on a different cluster, adapt `hydra.launcher`.
-
-Experiments with error status (or any other status) can be reset via:
-```bash
-python -m carps.experimenter.database.reset_experiments
-```
-
-### Get the results from the database and post-process
-
+`PyExperimenter`. See the
+[guide in the docs](https://automl.github.io/CARP-S/latest/guides/database/) for information
+about how to set it up.
 
 ## Adding a new Optimizer or Benchmark
 For instructions on how to add a new optimizer or benchmark, please refer to the contributing 
 
@@ -1,11 +1,12 @@
 # These have been configured to only really run short tasks. Longer form tasks
 # are usually completed in github actions.
+# Paths are relative to the Makefile location.
 
 SHELL := /bin/bash
 
 NAME := CARP-S
 PACKAGE_NAME := carps
-VERSION := 1.0.0
+VERSION := 1.0.1
 
 DIST := dist
 
@@ -36,7 +37,7 @@ check:
 	pre-commit run --all-files
 
 install-dev:
-	$(PIP) install -e ".[dev]"
+	$(PIP) install -e "../..[dev]"
 	pre-commit install
 
 clean-build:
@@ -80,33 +81,33 @@ uvenv:
 
 optimizer_smac:
 	$(PIP) install swig
-	$(PIP) install -r container_recipes/optimizers/SMAC3/SMAC3_requirements.txt
+	$(PIP) install -r ../container/recipes/optimizers/SMAC3/SMAC3_requirements.txt
 
 optimizer_optuna:
-	$(PIP) install -r container_recipes/optimizers/Optuna/Optuna_requirements.txt
+	$(PIP) install -r ../container/recipes/optimizers/Optuna/Optuna_requirements.txt
 
 optimizer_dehb:
-	$(PIP) install -r container_recipes/optimizers/DEHB/DEHB_requirements.txt
+	$(PIP) install -r ../container/recipes/optimizers/DEHB/DEHB_requirements.txt
 	$(PIP) install numpy --upgrade
 
 optimizer_skopt:
-	$(PIP) install -r container_recipes/optimizers/Scikit_Optimize/Scikit_Optimize_requirements.txt
+	$(PIP) install -r ../container/recipes/optimizers/Scikit_Optimize/Scikit_Optimize_requirements.txt
 
 optimizer_synetune:
-	$(PIP) install -r container_recipes/optimizers/SyneTune/SyneTune_requirements.txt
+	$(PIP) install -r ../container/recipes/optimizers/SyneTune/SyneTune_requirements.txt
 	$(PIP) install numpy --upgrade
 
 optimizer_ax:
-	$(PIP) install -r container_recipes/optimizers/Ax/Ax_requirements.txt
+	$(PIP) install -r ../container/recipes/optimizers/Ax/Ax_requirements.txt
 	$(PIP) install numpy --upgrade
 
 optimizer_hebo:
-	# . container_recipes/optimizers/HEBO/HEBO_install.sh
-	$(PIP) install -r container_recipes/optimizers/HEBO/HEBO_requirements.txt
+	# . ../container/recipes/optimizers/HEBO/HEBO_install.sh
+	$(PIP) install -r ../container/recipes/optimizers/HEBO/HEBO_requirements.txt
 	$(PIP) install numpy --upgrade
 
 optimizer_nevergrad:
-	$(PIP) install -r container_recipes/optimizers/Nevergrad/Nevergrad_requirements.txt
+	$(PIP) install -r ../container/recipes/optimizers/Nevergrad/Nevergrad_requirements.txt
 	$(PIP) install numpy --upgrade
 	$(PIP) install cma --upgrade
 
@@ -118,29 +119,29 @@ benchmark_bbob:
 benchmark_yahpo:
 # Needs 2GB of space for the surrogate models of YAHPO
 	# Install yahpo
-	. container_recipes/benchmarks/YAHPO/install_yahpo.sh
+	. ../container/recipes/benchmarks/YAHPO/install_yahpo.sh
 	$(PIP) install ConfigSpace --upgrade
 	$(PIP) install numpy --upgrade
 
 benchmark_pymoo:
 	# Install pymoo
-	$(PIP) install -r container_recipes/benchmarks/Pymoo/Pymoo_requirements.txt
+	$(PIP) install -r ../container/recipes/benchmarks/Pymoo/Pymoo_requirements.txt
 
 benchmark_mfpbench:
 	# Install mfpbench
-	$(PIP) install -r container_recipes/benchmarks/MFPBench/MFPBench_requirements.txt
+	$(PIP) install -r ../container/recipes/benchmarks/MFPBench/MFPBench_requirements.txt
 	$(PIP) install xgboost --upgrade
 	$(PIP) install ConfigSpace --upgrade
-	. container_recipes/benchmarks/MFPBench/download_data.sh
+	. ../container/recipes/benchmarks/MFPBench/download_data.sh
 
 benchmark_hpobench:
 	# Install hpobench
-	. container_recipes/benchmarks/HPOBench/install_HPOBench.sh
+	. ../container/recipes/benchmarks/HPOBench/install_HPOBench.sh
 
 benchmark_hpob:
 	# Install hpob
-	$(PIP) install -r container_recipes/benchmarks/HPOB/HPOB_requirements.txt
-	. container_recipes/benchmarks/HPOB/download_data.sh
+	$(PIP) install -r ../container/recipes/benchmarks/HPOB/HPOB_requirements.txt
+	. ../container/recipes/benchmarks/HPOB/download_data.sh
 
 benchmarks:
 	$(MAKE) benchmark_bbob
 
@@ -31,7 +31,7 @@ def run_make_commands(targets: list[str]) -> None:
 if __name__ == '__main__':
     args = sys.argv[1:]
     cwd_orig = os.getcwd()
-    makefile_dir = Path(os.path.dirname(__file__)).parent.parent
+    makefile_dir = Path(os.path.dirname(__file__))
     os.chdir(makefile_dir)
     run_make_commands(args)
     os.chdir(cwd_orig)
@@ -9,7 +9,7 @@ From: python:3.10-slim
 
 %files
     ./carps /benchmarking/carps
-    ./container_recipes /benchmarking/container_recipes
+    ./carps/container/recipes /benchmarking/carps/container/recipes
     requirements.txt /benchmarking/requirements.txt
     setup.py /benchmarking/setup.py
     README.md /benchmarking/README.md
@@ -27,13 +27,13 @@ From: python:3.10-slim
     pip install wheel
     pip install -r /benchmarking/requirements.txt
     pip install ../benchmarking
-    pip install -r /benchmarking/container_recipes/general/general_requirements_container_task.txt
+    pip install -r /benchmarking/carps/container/recipes/general/general_requirements_container_task.txt
 
     # log benchmarking version
     BENCHMARKING_VERSION=$(python -c "import carps; print(carps.version)")
     echo "benchmarking_version $BENCHMARKING_VERSION" >> "$SINGULARITY_LABELS"
 
     # benchmark-specific commands go here
-    pip install -r /benchmarking/container_recipes/benchmarks/BBOB/BBOB_requirements.txt
+    pip install -r /benchmarking/carps/container/recipes/benchmarks/BBOB/BBOB_requirements.txt
 
     echo "Successfully installed all features"