Merge branch 'master' of github.com:experimaestro/experimaestro-python

Author: bpiwowar

.github/workflows/pytest.yml

@@ -24,9 +24,11 @@ jobs:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
         run: |
-          python -m pip install --upgrade pip
+          # 1. Fresh tooling
+          python -m pip install -U pip setuptools wheel
+          # 2. Install pytest etc.
           pip install flake8 pytest pytest-timeout
-          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+          # 3. Install our package
           TEST_WORKFLOW=ON pip install .
       - name: Lint with flake8
         if: false

.readthedocs.yml

@@ -8,7 +8,7 @@ version: 2
 build:
   os: ubuntu-20.04
   tools:
-    python: "3.9"
+    python: "3.10"
 
 # Build documentation with MkDocs
 mkdocs:

docs/faq.md

@@ -24,3 +24,33 @@ task.instance(context).execute()
 ```
 
 The main problem with this approach is that resources are shared between experimaestro and the task
+
+### How to Debug a failed task ?
+If a task failed, you can rerun it with [debugpy](https://github.com/microsoft/debugpy). 
+
+#### Using vsCode
+If the task is already generated, you can run it with the [python debugger](https://code.visualstudio.com/docs/python/debugging) directly within vsCode.
+- open the task python file `.../HASHID/task_name.py`.
+- Run the dubugger Using the following configuration:
+
+In `.vscode/launch.json` :
+```json5
+ {
+            "name": "Python: XPM Task",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "experimaestro",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "args": [
+                "run",
+                "params.json"
+            ],
+            // "python": "${workspaceFolder}/.venv/bin/python",
+            "cwd": "${fileDirname}",
+            "env": {
+                "CUDA_VISIBLE_DEVICES": "1",
+            }
+}
+```
+- NOTE: if the task needs GPU support, you may need to open VS-Code on a node with access to a GPU.

docs/launchers/index.md

@@ -49,6 +49,23 @@ The `launchers.py` file dictates how a given *requirement* (e.g., 2 CPU with
 ::: experimaestro.launcherfinder.specs.CudaSpecification
 ::: experimaestro.launcherfinder.specs.CPUSpecification
 
+#### Parsing requirements
+
+::: experimaestro.launcherfinder.parser.parse
+
+```py3
+from experimaestro.launcherfinder.parse
+
+req = parse("""duration=40h & cpu(mem=700GiB) & cuda(mem=32GiB) * 8 | duration=50h & cpu(mem=700GiB) & cuda(mem=32GiB) * 4""")
+```
+
+Requirements can be manipulated:
+
+- duration can be multiplied by a given coefficient using
+  `req.multiply_duration`. For instance,  `req.multiply_duration(2)` multiplies
+  all the duration by 2.
+
+
 
 ### Example
 

pyproject.toml

@@ -1,6 +1,8 @@
-[tool.poetry]
+[project]
 name = "experimaestro"
-authors = ["Benjamin Piwowarski <benjamin@piwowarski.fr>"]
+authors = [
+    {name = "Benjamin Piwowarski", email = "benjamin@piwowarski.fr"}
+]
 description = '"Experimaestro is a computer science experiment manager"'
 readme = "README.md"
 license = "GPL-3"
@@ -14,15 +16,38 @@ classifiers = [
     "Programming Language :: Python :: 3",
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
-include = [
-    "src/experimaestro/server/data/*",
-    "src/experimaestro/sphinx/static/experimaestro.css",
-    "src/experimaestro/mkdocs/style.css",
-    { path="src/experimaestro/server/data/*", format=['sdist', 'wheel']}
-]
-version = "0.0.0"
 repository = "https://github.com/experimaestro/experimaestro-python"
 documentation = "https://experimaestro-python.readthedocs.io/"
+dynamic = ["version"]
+requires-python = ">=3.10"
+dependencies = [
+    "arpeggio >=2,<3",
+    "attrs >=23.1.0,<24",
+    "click >=8",
+    "decorator >=5,<6",
+    "docstring-parser >=0.15,<1",
+    "fasteners >=0.19,<1",
+    "flask >=2.3,<3",
+    "flask-socketio >=5.3,<6",
+    "gevent >=25",
+    "gevent-websocket >=0.10",
+    "humanfriendly >=10",
+    "huggingface-hub >0.17",
+    "marshmallow >=3.20,<4",
+    "mkdocs >=1.5,<2",
+    "omegaconf >=2.3,<3",
+    "psutil >=7,<8",
+    "pyparsing >=3.1,<4",
+    "pytools >=2023.1.1,<2024",
+    "pyyaml >=6.0.1,<7",
+    "requests >=2.31,<3",
+    "rpyc >=5,<7",
+    "sortedcontainers >=2.4,<3",
+    "termcolor >=2.3,<3",
+    "tqdm >=4.66.1,<5",
+    "typing-extensions >=4.2; python_version < \"3.12\"",
+    "watchdog >=2"
+]
 
 [tool.poetry-dynamic-versioning]
 enable = true
@@ -33,38 +58,31 @@ format-jinja = """{%- set pre = [] -%}{%- set metadata = [] -%}
 {%- if revision is not none -%}{{ pre.append("rc" + revision|string) or "" }}{%- endif -%}
 {%- if distance > 0 -%}{{ metadata.append(distance|string) or "" }}{%- endif -%}
 {{ serialize_semver(base, pre, metadata)}}"""
+
+[tool.poetry]
+include = [
+    "src/experimaestro/server/data/*",
+    "src/experimaestro/sphinx/static/experimaestro.css",
+    "src/experimaestro/mkdocs/style.css",
+    { path="src/experimaestro/server/data/*", format=['sdist', 'wheel']}
+]
+version = "0.0.0"
+
+[tool.poetry-dynamic-versioning.files."src/experimaestro/version.py"]
+persistent-substitution = true
+initial-content = """
+  # These version placeholders will be replaced later during substitution.
+  __version__ = "0.0.0"
+  __version_tuple__ = (0, 0, 0)
+"""
+
+[tool.poetry.requires-plugins]
+poetry-dynamic-versioning = { version = ">=1.0.0,<2.0.0", extras = ["plugin"] }
+
 [build-system]
 requires = ["poetry-core>=1.0.0", "poetry-dynamic-versioning>=1.0.0,<2.0.0"]
 build-backend = "poetry_dynamic_versioning.backend"
 
-[tool.poetry.dependencies]
-python = "^3.9"
-click = ">=8"
-omegaconf = "^2.3"
-typing-extensions = {version = ">=4.2", markers = "python_version < \"3.12\""}
-attrs = "^23.1.0"
-fasteners = "^0.19"
-pyyaml = "^6.0.1"
-psutil = ">=7"
-pytools = "^2023.1.1"
-tqdm = "^4.66.1"
-docstring-parser = "^0.15"
-termcolor = ">=2.3"
-requests = "^2.31"
-sortedcontainers = "^2.4"
-pyparsing = "^3.1"
-humanfriendly = "^10"
-huggingface-hub = ">0.17"
-gevent = "^24.11.1"
-gevent-websocket = "^0.10"
-flask = "^2.3"
-flask-socketio = "^5.3"
-arpeggio = "^2"
-watchdog = "^2"
-marshmallow = "^3.20"
-decorator = "^5"
-rpyc = ">=5,<7"
-
 [tool.poetry.group.ssh]
 optional = true
 
@@ -79,21 +97,21 @@ optional = true
 docutils = "^0.18"
 Pygments = "^2.15"
 
-[tool.poetry.scripts]
+[project.scripts]
 experimaestro = "experimaestro.__main__:main"
 
-[tool.poetry.plugins."mkdocs.plugins"]
+[project.entry-points."mkdocs.plugins"]
 experimaestro = "experimaestro.mkdocs:Documentation"
 
-[tool.poetry.plugins."experimaestro.process"]
+[project.entry-points."experimaestro.process"]
 local = "experimaestro.connectors.local:LocalProcess"
 slurm = "experimaestro.launchers.slurm:BatchSlurmProcess"
 
-[tool.poetry.plugins."experimaestro.connectors"]
+[project.entry-points."experimaestro.connectors"]
 local = "experimaestro.connectors.local:LocalConnector"
 ssh = "experimaestro.connectors.ssh:SshConnector"
 
-[tool.poetry.plugins."experimaestro.tokens"]
+[project.entry-points."experimaestro.tokens"]
 unix = "experimaestro.tokens:CounterToken"
 
 
@@ -122,7 +140,7 @@ warn_unused_ignores = true
 
 [tool.commitizen]
 name = "cz_conventional_commits"
-version = "1.9.1"
+version = "1.11.0"
 changelog_start_rev = "v1.0.0"
 tag_format = "v$major.$minor.$patch$prerelease"
 # update_changelog_on_bump = true

src/experimaestro/launcherfinder/parser.py

@@ -51,7 +51,12 @@ def cpu():
 
 
 def duration():
-    return "duration", "=", RegExMatch(r"\d+"), RegExMatch(r"h(ours)?|d(ays)?")
+    return (
+        "duration",
+        "=",
+        RegExMatch(r"\d+"),
+        RegExMatch(r"h(ours?)?|d(ays?)?|m(ins?)?"),
+    )
 
 
 def one_spec():
@@ -67,7 +72,7 @@ def grammar():
 
 class Visitor(PTNodeVisitor):
     def visit_grammar(self, node, children):
-        return [child for child in children]
+        return specs.RequirementUnion(*[child for child in children])
 
     def visit_one_spec(self, node, children):
         return reduce(lambda x, el: x & el, children)

src/experimaestro/launcherfinder/registry.py

@@ -9,7 +9,7 @@
 import pkg_resources
 from experimaestro.utils import logger
 from .base import ConnectorConfiguration, TokenConfiguration
-from .specs import HostRequirement
+from .specs import HostRequirement, RequirementUnion
 
 if typing.TYPE_CHECKING:
     from experimaestro.launchers import Launcher
@@ -150,18 +150,18 @@ def find(
         # Parse specs
         from .parser import parse
 
-        specs = []
+        specs = RequirementUnion()
         for spec in input_specs:
             if isinstance(spec, str):
-                specs.extend(parse(spec))
+                specs.add(parse(spec))
             else:
-                specs.append(spec)
+                specs.add(spec)
 
         # Use launcher function
         from experimaestro.launchers import Launcher
 
         if self.find_launcher_fn is not None:
-            for spec in specs:
+            for spec in specs.requirements:
                 if launcher := self.find_launcher_fn(spec, tags):
                     assert isinstance(
                         launcher, Launcher

src/experimaestro/launcherfinder/specs.py

@@ -1,4 +1,6 @@
+from abc import ABC, abstractmethod
 import logging
+import math
 from attr import Factory
 from attrs import define
 from copy import copy, deepcopy
@@ -95,7 +97,7 @@ class MatchRequirement:
     requirement: "HostSimpleRequirement"
 
 
-class HostRequirement:
+class HostRequirement(ABC):
     """A requirement must be a disjunction of host requirements"""
 
     requirements: List["HostSimpleRequirement"]
@@ -110,6 +112,12 @@ def __or__(self, other: "HostRequirement"):
     def match(self, host: HostSpecification) -> Optional[MatchRequirement]:
         raise NotImplementedError()
 
+    @abstractmethod
+    def multiply_duration(self, coefficient: float) -> "HostRequirement":
+        """Returns a new HostRequirement with a duration multiplied by the
+        provided coefficient"""
+        ...
+
 
 class RequirementUnion(HostRequirement):
     """Ordered list of simple host requirements -- the first one is the priority"""
@@ -119,6 +127,16 @@ class RequirementUnion(HostRequirement):
     def __init__(self, *requirements: "HostSimpleRequirement"):
         self.requirements = list(requirements)
 
+    def add(self, requirement: "HostRequirement"):
+        match requirement:
+            case HostSimpleRequirement():
+                self.requirements.extend(*requirement.requirements)
+            case RequirementUnion():
+                self.requirements.append(requirement)
+            case _:
+                raise RuntimeError("Cannot handle type %s", type(requirement))
+        return self
+
     def match(self, host: HostSpecification) -> Optional[MatchRequirement]:
         """Returns the matched requirement (if any)"""
 
@@ -133,6 +151,14 @@ def match(self, host: HostSpecification) -> Optional[MatchRequirement]:
 
         return argmax
 
+    def multiply_duration(self, coefficient: float) -> "RequirementUnion":
+        return RequirementUnion(
+            *[r.multiply_duration(coefficient) for r in self.requirements]
+        )
+
+    def __repr__(self):
+        return " | ".join(repr(r) for r in self.requirements)
+
 
 class HostSimpleRequirement(HostRequirement):
     """Simple host requirement"""
@@ -149,6 +175,11 @@ class HostSimpleRequirement(HostRequirement):
     def __repr__(self):
         return f"Req(cpu={self.cpu}, cuda={self.cuda_gpus}, duration={self.duration})"
 
+    def multiply_duration(self, coefficient: float) -> "HostSimpleRequirement":
+        r = HostSimpleRequirement(self)
+        r.duration = math.ceil(self.duration * coefficient)
+        return r
+
     def __init__(self, *reqs: "HostSimpleRequirement"):
         self.cuda_gpus = []
         self.cpu = CPUSpecification(0, 0)