From 890a8c8c5f5f36a6f6dac167e2cee1521ddd1e7a Mon Sep 17 00:00:00 2001 From: Sabine Date: Thu, 15 Aug 2024 15:56:12 +0000 Subject: [PATCH 01/14] list methods in class docstring --- src/neptune_scale/__init__.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/neptune_scale/__init__.py b/src/neptune_scale/__init__.py index ec1a4127..70a90483 100644 --- a/src/neptune_scale/__init__.py +++ b/src/neptune_scale/__init__.py @@ -50,6 +50,10 @@ class Run(WithResources, AbstractContextManager): """ Representation of tracked metadata. + + Methods: + close(): Synchronizes all remaining data and closes the connection to Neptune. + log(): Logs the specified metadata to Neptune. """ def __init__( From 4a5d79ecb24a00585752f01eb7176a51719c9b20 Mon Sep 17 00:00:00 2001 From: Sabine Date: Thu, 15 Aug 2024 16:00:23 +0000 Subject: [PATCH 02/14] tweak docstring for Run.__init__ --- src/neptune_scale/__init__.py | 61 +++++++++++++++++++++++++++-------- 1 file changed, 47 insertions(+), 14 deletions(-) diff --git a/src/neptune_scale/__init__.py b/src/neptune_scale/__init__.py index 70a90483..8123bda9 100644 --- a/src/neptune_scale/__init__.py +++ b/src/neptune_scale/__init__.py @@ -75,23 +75,56 @@ def __init__( Initializes a run that logs the model-building metadata to Neptune. Args: - family: Identifies related runs. For example, the same value must apply to all runs within a run hierarchy. + family (str): Identifies related runs. All runs of the same lineage must have the same `family` value. Max length: 128 characters. - run_id: Unique identifier of a run. Must be unique within the project. Max length: 128 characters. - project: Name of the project where the metadata is logged, in the form `workspace-name/project-name`. + run_id (str): Identifier of a run. Must be unique within the project. Max length: 128 characters. + project (str): Name of the project where the metadata is logged, in the form `workspace-name/project-name`. If not provided, the value of the `NEPTUNE_PROJECT` environment variable is used. - api_token: Your Neptune API token. If not provided, the value of the `NEPTUNE_API_TOKEN` environment + api_token (str): Your Neptune API token. If not provided, the value of the `NEPTUNE_API_TOKEN` environment variable is used. - resume: Whether to resume an existing run. - as_experiment: If creating a run as an experiment, ID of an experiment to be associated with the run. - creation_time: Custom creation time of the run. - from_run_id: If forking from an existing run, ID of the run to fork from. - from_step: If forking from an existing run, step number to fork from. - max_queue_size: Maximum number of operations in a queue. - max_queue_size_exceeded_callback: Callback function triggered when a queue is full. - Accepts two arguments: - - Maximum size of the queue. - - Exception that made the queue full. + resume (bool): If `False` (default), creates a new run. To continue an existing run, set to `True` and pass + the ID of an existing run to the `run_id` argument. + To fork a run, use `from_run_id` and `from_step` instead. + as_experiment (str): To mark a run as an experiment, pass an experiment ID. + creation_time (datetime): Custom creation time of the run. + from_run_id (str): To forking off an existing run, pass the ID of the run to fork from. + from_step (int): To fork off an existing run, pass the step number to fork from. + max_queue_size (int): Maximum number of operations allowed in the queue. + max_queue_size_exceeded_callback (Callable[[int, BaseException], None]): Callback function triggered when + the queue is full. The function should take two arguments: + 1. Maximum size of the queue. + 2. Exception that made the queue full. + + Examples: + + Create a new run: + + ``` + from neptune_scale import Run + + with Run( + project="team-alpha/project-x", + api_token="h0dHBzOi8aHR0cHM6...Y2MifQ==", + family="aquarium", + run_id="likable-barracuda", + ) as run: + ... + ``` + + Create a forked run and mark it as an experiment: + + ``` + from neptune_scale import Run + + with Run( + family="aquarium", + run_id="adventurous-barracuda", + as_experiment="swim-further", + from_run_id="likable-barracuda", + from_step=102, + ) as run: + ... + ``` """ verify_type("family", family, str) verify_type("run_id", run_id, str) From 339c0888af49d3a3bbfddb365f435f6c3d565fdc Mon Sep 17 00:00:00 2001 From: Sabine Date: Wed, 28 Aug 2024 08:59:50 +0000 Subject: [PATCH 03/14] temp precommit workaround --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index f5fab8f8..b0d69ca7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -90,3 +90,4 @@ check_untyped_defs = "True" warn_return_any = "True" show_error_codes = "True" warn_unused_ignores = "True" +disable_error_code = "attr-defined" From 5fe6fb7e2ef62bd68bcddd496943b52c53564105 Mon Sep 17 00:00:00 2001 From: Sabine Date: Wed, 28 Aug 2024 10:00:31 +0000 Subject: [PATCH 04/14] tweak log() docstring --- src/neptune_scale/__init__.py | 34 +++++++++++++++++++++++++--------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/src/neptune_scale/__init__.py b/src/neptune_scale/__init__.py index 9f7395e0..c5419fbe 100644 --- a/src/neptune_scale/__init__.py +++ b/src/neptune_scale/__init__.py @@ -340,22 +340,38 @@ def log( remove_tags: Optional[Dict[str, Union[List[str], Set[str]]]] = None, ) -> None: """ - Logs the specified metadata to Neptune. + Logs the specified metadata to a Neptune run. + + You can log metrics, tags, and configurations. Pass the metadata as a dictionary {key: value} with + + - key: path to where the metadata should be stored in the run. + - value: the piece of metadata to log. + + For example, log(fields={"parameters/learning_rate": 0.001}) + In the field path, each forward slash "/" nests the field under a namespace. + Use namespaces to structure the metadata into meaningful categories. Args: - step: Index of the log entry, must be increasing. If None, the highest of the already logged indexes is used. + step: Index of the log entry. Must be increasing. + If None, the highest of the already logged indexes is used. timestamp: Time of logging the metadata. - fields: Dictionary of fields to log. - metrics: Dictionary of metrics to log. - add_tags: Dictionary of tags to add to the run. - remove_tags: Dictionary of tags to remove from the run. + fields: Dictionary of configs or other values to log. Independent of the step value. + Available types: float, integer, Boolean, string, and datetime. + To log multiple values at once, pass multiple dictionaries. + metrics: Dictionary of metrics to log. Each metric value is associated with a step. + To log multiple metrics at once, pass multiple dictionaries. + Each metric is represented as a series of float values in the run. + add_tags: Dictionary of tags to add to the run, as a list of strings. Independent of the step value. + remove_tags: Dictionary of tags to remove from the run, as a list of strings. Independent of the step value. Examples: ``` + >>> from neptune_scale import Run >>> with Run(...) as run: - ... run.log(step=1, fields={"parameters/learning_rate": 0.001}) - ... run.log(step=2, add_tags={"sys/group_tags": ["group1", "group2"]}) - ... run.log(step=3, metrics={"metrics/loss": 0.1}) + ... run.log(fields={"parameters/learning_rate": 0.001}) + ... run.log(add_tags={"sys/tags": ["tag1", "tag2"]}) + ... run.log(step=1, metrics={"loss": 0.11, "acc": 0.81}) + >>> run.log(step=2, metrics={"loss": 0.09, "acc": 0.82}) ``` """ From 583b6e4ee80d9800b0dca313ca84bb2aa6c346d2 Mon Sep 17 00:00:00 2001 From: Sabine Date: Wed, 28 Aug 2024 10:01:06 +0000 Subject: [PATCH 05/14] revert precommit workaround --- pyproject.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 620aa000..4850781f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -94,4 +94,3 @@ check_untyped_defs = "True" warn_return_any = "True" show_error_codes = "True" warn_unused_ignores = "True" -disable_error_code = "attr-defined" From a6a484ef3140bc5c13d39937c20099591e6cd6e4 Mon Sep 17 00:00:00 2001 From: Sabine Date: Wed, 28 Aug 2024 11:47:54 +0000 Subject: [PATCH 06/14] create README --- README.md | 254 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 251 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index e3dde645..e678b1ae 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,251 @@ -### Local Environment -Create new virtual environment and run `pip install -r dev_requirements.txt` -Run `pre-commit install` to initialize [pre-commit](https://pre-commit.com/) +# Neptune Scale client + +> [!NOTE] +> This package only works with the `3.0` version of neptune.ai called Neptune Scale, which is in beta. +> +> You can't use the Scale client with the stable Neptune `2.x` versions currently available to SaaS and self-hosting customers. For the Python client corresponding to Neptune `2.x`, see https://github.com/neptune-ai/neptune-client. + +**What is Neptune?** + +Neptune is an experiment tracker. It enables researchers to monitor their model training, visualize and compare model metadata, and collaborate on AI/ML projects within a team. + +**What's different about Neptune Scale?** + +Neptune Scale is the next major version of Neptune. It's built on an entirely new architecture for ingesting and rendering data, with a focus on responsiveness and accuracy at scale. + +Neptune Scale supports forked experiments, with built-in mechanics for retaining run ancestry. This way, you can focus on analyzing the latest runs, but also visualize the full history of your experiments. + +## Installation + +```bash +pip install neptune-client-scale +``` + +## Example usage + +```python +from neptune_scale import Run + +run = Run( + family="RunFamilyName", + run_id="SomeUniqueRunIdentifier", +) + +run.log(metrics={"MetricName": metric_value}) + +run.close() +``` + +## API reference + +### `Run` + +Representation of experiment tracking metadata logged with Neptune Scale. + +#### Initialization + +Initialize with the class constructor: + +```python +from neptune_scale import Run + +run = Run(...) +``` + +or using a context manager: + +```python +from neptune_scale import Run + +with Run(...) as run: + ... +``` + +__Parameters__ + +| Name | Type | Default | Description | +|------------------|------------------|---------|---------------------------------------------------------------------------| +| `family` | `str` | - | Identifies related runs. All runs of the same lineage must have the same `family` value, i.e forking is only possible within the same family. Max length: 128 characters. | +| `run_id` | `str` | `None` | Identifier of the run. Must be unique within the project. Max length: 128 characters. | +| `project` | `str`, optional | `None` | Name of a project in the form `workspace-name/project-name`. If `None`, the value of the `NEPTUNE_PROJECT` environment variable is used. | +| `api_token` | `str`, optional | `None` | Your Neptune API token (or a service account's API token). If `None`, the value of the `NEPTUNE_API_TOKEN` environment variable is used. To keep your token secure, avoid placing it in source code. Instead, save it as an environment variable. | +| `resume` | `bool`, optional | `False` | If `False` (default), creates a new run. To continue an existing run, set to `True` and pass the ID of an existing run to the `run_id` argument. To fork a run, use `from_run_id` and `from_step` instead. | +| `mode` | `Literal`, `"async"` or `"disabled"` | `"async"` | Mode of operation. If set to `"disabled"`, the run doesn't log any metadata. | +| `as_experiment` | `str`, optional | `None` | Name of the experiment to associate the run with. | +| `creation_time` | `datetime`, optional | `None` | Custom creation time of the run. | +| `from_run_id` | `str`, optional | `None` | If forking off an existing run, ID of the run to fork from. | +| `from_step` | `int`, optional | - | If forking off an existing run, step number to fork from. | +| `max_queue_size` | `int`, optional | `None` | Maximum number of operations allowed in the queue. | +| `on_queue_full_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when the queue is full. The function should take two arguments: (1) Exception that made the queue full. (2) Optional timestamp: When the exception was last raised. | +| `on_network_error_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when a network error occurs. | +| `on_error_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | The default callback function triggered when an error occurs. Applies if an error wasn't caught by other callbacks. | +| `on_warning_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when a warning occurs. | + +__Examples__ + +Create a new run: + +```python +from neptune_scale import Run + +with Run( + project="team-alpha/project-x", + api_token="h0dHBzOi8aHR0cHM6...Y2MifQ==", + family="aquarium", + run_id="likable-barracuda", +) as run: + ... +``` + +> [!TIP] +> Find your API token in your user menu, in the bottom-left corner of the Neptune app. +> +> Or, to use shared API tokens for multiple users or non-human accounts, create a service account in your workspace settings. + +Create a forked run and mark it as an experiment: + +```python +with Run( + family="aquarium", + run_id="adventurous-barracuda", + as_experiment="swim-further", + from_run_id="likable-barracuda", + from_step=102, +) as run: + ... +``` + +Continue a run: + +```python +with Run( + family="aquarium", + run_id="likable-barracuda", # a Neptune run with this ID already exists + resume=True, +) as run: + ... +``` + +## `close()` + +Stops the connection to Neptune and synchronizes all data. + +__Examples__ + +```python +from neptune_scale import Run + +run = Run(...) +run.log(...) + +run.close() +``` + +If using a context manager, Neptune automatically closes the run upon exiting the context: + +```python +with Run(...) as run: + ... + +# run is closed at the end of the context +``` + +## `log()` + +Logs the specified metadata to a Neptune run. + +You can log metrics, tags, and configurations. Pass the metadata as a dictionary `{key: value}` with + +- `key`: path to where the metadata should be stored in the run. +- `value`: the piece of metadata to log. + +For example, `{"parameters/learning_rate": 0.001}`. In the field path, each forward slash `/` nests the field under a namespace. Use namespaces to structure the metadata into meaningful categories. + +__Parameters__ + +| Name | Type | Default | Description | +|---------------|----------------------------------------------------|---------|---------------------------------------------------------------------------| +| `step` | `Union[float, int]`, optional | `None` | Index of the log entry. Must be increasing. If `None`, the highest of the already logged indexes is used. | +| `timestamp` | `datetime`, optional | `None` | Time of logging the metadata. | +| `fields` | `Dict[str, Union[float, bool, int, str, datetime, list, set]]`, optional | `None` | Dictionary of configs or other values to log. Independent of the step value. Available types: float, integer, Boolean, string, and datetime. To log multiple values at once, pass multiple dictionaries. | +| `metrics` | `Dict[str, float]`, optional | `None` | Dictionary of metrics to log. Each metric value is associated with a step. To log multiple metrics at once, pass multiple dictionaries. | +| `add_tags` | `Dict[str, Union[List[str], Set[str]]]`, optional | `None` | Dictionary of tags to add to the run, as a list of strings. Independent of the step value. | +| `remove_tags` | `Dict[str, Union[List[str], Set[str]]]`, optional | `None` | Dictionary of tags to remove from the run, as a list of strings. Independent of the step value. | + +__Examples__ + +Create a run and log some metadata: + +```python +from neptune_scale import Run + +with Run(...) as run: + run.log(fields={"parameters/learning_rate": 0.001}) + run.log(add_tags={"sys/tags": ["tag1", "tag2"]}) + run.log(metrics={"loss": 0.14, "acc": 0.78}) +``` + +You can explicitly pass the step when logging metrics: + +```python +run.log(step=5, metrics={"loss": 0.09, "acc": 0.82}) # works if the previous step is no higher than 4 +run.log(metrics={"loss": 0.08, "acc": 0.86}) # step is set to "6" +... +``` + +Remove a tag: + +```python +with Run(...) as run: + run.log(remove_tags={"sys/tags": "tag2"}) +``` + +## `wait_for_submission()` + +Waits until all metadata is submitted to Neptune. + +__Parameters__ + +| Name | Type | Default | Description | +|-----------|-------------------|---------|---------------------------------------------------------------------------| +| `timeout` | `float`, optional | `None` | In seconds, the maximum time to wait for submission. | +| `verbose` | `bool`, optional | `True` | If True (default), prints messages about the waiting process. | + +__Example__ + +```python +from neptune_scale import Run + +with Run(...) as run: + run.log(...) + ... + run.wait_for_submission() + run.log(fields={"scores/some_score": some_score_value}) # called once queued Neptune operations have been submitted +``` + +## `wait_for_processing()` + +Waits until all metadata is processed by Neptune. + +__Parameters__ + +| Name | Type | Default | Description | +|-----------|-------------------|---------|---------------------------------------------------------------------------| +| `timeout` | `float`, optional | `None` | In seconds, the maximum time to wait for processing. | +| `verbose` | `bool`, optional | `True` | If True (default), prints messages about the waiting process. | + +__Example__ + +```python +from neptune_scale import Run + +with Run(...) as run: + run.log(...) + ... + run.wait_for_processing() + run.log(fields={"scores/some_score": some_score_value}) # called once queued Neptune operations have been processed +``` + +## Getting help + +For questions or comments, contact support@neptune.ai. From c757b705b04711217f23a3275df4ebd2cb86fde2 Mon Sep 17 00:00:00 2001 From: Sabine Date: Wed, 28 Aug 2024 11:52:15 +0000 Subject: [PATCH 07/14] fix run_id default in API reference --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e678b1ae..2c60fef1 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ __Parameters__ | Name | Type | Default | Description | |------------------|------------------|---------|---------------------------------------------------------------------------| | `family` | `str` | - | Identifies related runs. All runs of the same lineage must have the same `family` value, i.e forking is only possible within the same family. Max length: 128 characters. | -| `run_id` | `str` | `None` | Identifier of the run. Must be unique within the project. Max length: 128 characters. | +| `run_id` | `str` | - | Identifier of the run. Must be unique within the project. Max length: 128 characters. | | `project` | `str`, optional | `None` | Name of a project in the form `workspace-name/project-name`. If `None`, the value of the `NEPTUNE_PROJECT` environment variable is used. | | `api_token` | `str`, optional | `None` | Your Neptune API token (or a service account's API token). If `None`, the value of the `NEPTUNE_API_TOKEN` environment variable is used. To keep your token secure, avoid placing it in source code. Instead, save it as an environment variable. | | `resume` | `bool`, optional | `False` | If `False` (default), creates a new run. To continue an existing run, set to `True` and pass the ID of an existing run to the `run_id` argument. To fork a run, use `from_run_id` and `from_step` instead. | From 7fc22694ce88fbe8db72142dc85dcb72b5e8af97 Mon Sep 17 00:00:00 2001 From: Sabine Date: Wed, 28 Aug 2024 11:54:22 +0000 Subject: [PATCH 08/14] fix from_stepd default in API reference --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 2c60fef1..b9bd7c12 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,7 @@ __Parameters__ | Name | Type | Default | Description | |------------------|------------------|---------|---------------------------------------------------------------------------| | `family` | `str` | - | Identifies related runs. All runs of the same lineage must have the same `family` value, i.e forking is only possible within the same family. Max length: 128 characters. | -| `run_id` | `str` | - | Identifier of the run. Must be unique within the project. Max length: 128 characters. | +| `run_id` | `str` | - | Identifier of the run. Must be unique within the project. Max length: 128 characters. | | `project` | `str`, optional | `None` | Name of a project in the form `workspace-name/project-name`. If `None`, the value of the `NEPTUNE_PROJECT` environment variable is used. | | `api_token` | `str`, optional | `None` | Your Neptune API token (or a service account's API token). If `None`, the value of the `NEPTUNE_API_TOKEN` environment variable is used. To keep your token secure, avoid placing it in source code. Instead, save it as an environment variable. | | `resume` | `bool`, optional | `False` | If `False` (default), creates a new run. To continue an existing run, set to `True` and pass the ID of an existing run to the `run_id` argument. To fork a run, use `from_run_id` and `from_step` instead. | @@ -74,7 +74,7 @@ __Parameters__ | `as_experiment` | `str`, optional | `None` | Name of the experiment to associate the run with. | | `creation_time` | `datetime`, optional | `None` | Custom creation time of the run. | | `from_run_id` | `str`, optional | `None` | If forking off an existing run, ID of the run to fork from. | -| `from_step` | `int`, optional | - | If forking off an existing run, step number to fork from. | +| `from_step` | `int`, optional | `None` | If forking off an existing run, step number to fork from. | | `max_queue_size` | `int`, optional | `None` | Maximum number of operations allowed in the queue. | | `on_queue_full_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when the queue is full. The function should take two arguments: (1) Exception that made the queue full. (2) Optional timestamp: When the exception was last raised. | | `on_network_error_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when a network error occurs. | From 6cca3d50878df29daf1ba5a206e05aa2ac3aa992 Mon Sep 17 00:00:00 2001 From: Sabine Date: Thu, 29 Aug 2024 09:14:59 +0000 Subject: [PATCH 09/14] Apply suggestions from code review Co-authored-by: Krzysztof Godlewski --- README.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index b9bd7c12..7382fdc2 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,10 @@ run = Run( run_id="SomeUniqueRunIdentifier", ) -run.log(metrics={"MetricName": metric_value}) +run.log( + metrics={"Metric1": metric1_value, "Metric2": metric2_value}, + fields={"Field1": field1_value} +) run.close() ``` @@ -78,7 +81,7 @@ __Parameters__ | `max_queue_size` | `int`, optional | `None` | Maximum number of operations allowed in the queue. | | `on_queue_full_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when the queue is full. The function should take two arguments: (1) Exception that made the queue full. (2) Optional timestamp: When the exception was last raised. | | `on_network_error_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when a network error occurs. | -| `on_error_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | The default callback function triggered when an error occurs. Applies if an error wasn't caught by other callbacks. | +| `on_error_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | The default callback function triggered when an unrecoverable error occurs. Applies if an error wasn't caught by other callbacks. In this callback you can choose to perform your cleanup operations and close the training script. | | `on_warning_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when a warning occurs. | __Examples__ From 34fbe71b1eab484debf46b3cba6d575f6c13faed Mon Sep 17 00:00:00 2001 From: Sabine Date: Thu, 29 Aug 2024 09:26:45 +0000 Subject: [PATCH 10/14] add link to experiments documentation --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 7382fdc2..20de8710 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ __Parameters__ | `api_token` | `str`, optional | `None` | Your Neptune API token (or a service account's API token). If `None`, the value of the `NEPTUNE_API_TOKEN` environment variable is used. To keep your token secure, avoid placing it in source code. Instead, save it as an environment variable. | | `resume` | `bool`, optional | `False` | If `False` (default), creates a new run. To continue an existing run, set to `True` and pass the ID of an existing run to the `run_id` argument. To fork a run, use `from_run_id` and `from_step` instead. | | `mode` | `Literal`, `"async"` or `"disabled"` | `"async"` | Mode of operation. If set to `"disabled"`, the run doesn't log any metadata. | -| `as_experiment` | `str`, optional | `None` | Name of the experiment to associate the run with. | +| `as_experiment` | `str`, optional | `None` | Name of the experiment to associate the run with. Learn more about [experiments](https://docs-beta.neptune.ai/concepts) in the Neptune documentation. | | `creation_time` | `datetime`, optional | `None` | Custom creation time of the run. | | `from_run_id` | `str`, optional | `None` | If forking off an existing run, ID of the run to fork from. | | `from_step` | `int`, optional | `None` | If forking off an existing run, step number to fork from. | From 1ba43c861ae71b276751196e746b9f81ebf55530 Mon Sep 17 00:00:00 2001 From: Sabine Date: Thu, 29 Aug 2024 10:37:00 +0000 Subject: [PATCH 11/14] address review comments --- README.md | 40 +++++++++++++++++++++++++--------------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/README.md b/README.md index 20de8710..99c3dcdf 100644 --- a/README.md +++ b/README.md @@ -78,7 +78,7 @@ __Parameters__ | `creation_time` | `datetime`, optional | `None` | Custom creation time of the run. | | `from_run_id` | `str`, optional | `None` | If forking off an existing run, ID of the run to fork from. | | `from_step` | `int`, optional | `None` | If forking off an existing run, step number to fork from. | -| `max_queue_size` | `int`, optional | `None` | Maximum number of operations allowed in the queue. | +| `max_queue_size` | `int`, optional | 1M | Maximum number of operations queued for processing. 1 000 000 by default. You should raise this value if you see the `on_queue_full_callback` function being called. | | `on_queue_full_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when the queue is full. The function should take two arguments: (1) Exception that made the queue full. (2) Optional timestamp: When the exception was last raised. | | `on_network_error_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when a network error occurs. | | `on_error_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | The default callback function triggered when an unrecoverable error occurs. Applies if an error wasn't caught by other callbacks. In this callback you can choose to perform your cleanup operations and close the training script. | @@ -131,7 +131,9 @@ with Run( ## `close()` -Stops the connection to Neptune and synchronizes all data. +Waits for all locally queued data to be processed by Neptune (see [`wait_for_processing()`](#wait_for_processing)) and closes the run. + +This is a blocking operation. You should call the function at the end of your script, after your model training is completed. __Examples__ @@ -168,7 +170,7 @@ __Parameters__ | Name | Type | Default | Description | |---------------|----------------------------------------------------|---------|---------------------------------------------------------------------------| -| `step` | `Union[float, int]`, optional | `None` | Index of the log entry. Must be increasing. If `None`, the highest of the already logged indexes is used. | +| `step` | `Union[float, int]`, optional | `None` | Index of the log entry. Must be increasing. If not specified, the `log()` call increments the step starting from the highest already logged value. **Tip:** Using float rather than int values can be useful, for example, when logging substeps in a batch. | | `timestamp` | `datetime`, optional | `None` | Time of logging the metadata. | | `fields` | `Dict[str, Union[float, bool, int, str, datetime, list, set]]`, optional | `None` | Dictionary of configs or other values to log. Independent of the step value. Available types: float, integer, Boolean, string, and datetime. To log multiple values at once, pass multiple dictionaries. | | `metrics` | `Dict[str, float]`, optional | `None` | Dictionary of metrics to log. Each metric value is associated with a step. To log multiple metrics at once, pass multiple dictionaries. | @@ -183,29 +185,35 @@ Create a run and log some metadata: from neptune_scale import Run with Run(...) as run: - run.log(fields={"parameters/learning_rate": 0.001}) - run.log(add_tags={"sys/tags": ["tag1", "tag2"]}) - run.log(metrics={"loss": 0.14, "acc": 0.78}) + run.log( + fields={"parameters/learning_rate": 0.001}, + add_tags={"sys/tags": ["tag1", "tag2"]}, + metrics={"loss": 0.14, "acc": 0.78}, + ) ``` -You can explicitly pass the step when logging metrics: +Remove a tag: ```python -run.log(step=5, metrics={"loss": 0.09, "acc": 0.82}) # works if the previous step is no higher than 4 -run.log(metrics={"loss": 0.08, "acc": 0.86}) # step is set to "6" -... +with Run(...) as run: + run.log(remove_tags={"sys/tags": "tag2"}) ``` -Remove a tag: +You can pass the step when logging metrics: ```python -with Run(...) as run: - run.log(remove_tags={"sys/tags": "tag2"}) +run.log(step=5, metrics={"loss": 0.09, "acc": 0.82}) # works if the previous step is no higher than 4 +run.log(metrics={"loss": 0.08, "acc": 0.86}) # step index is set to "6" +... ``` +**Note:** Calling `log()` without specifying the step still increments the index. To correlate logged values, make sure to send all metadata related to a step in a single `log()` call, or specify the step explicitly. + ## `wait_for_submission()` -Waits until all metadata is submitted to Neptune. +Waits until all metadata is submitted to Neptune for processing. + +When submitted, the data is not yet saved in Neptune (see [`wait_for_processing()`](#wait_for_processing)). __Parameters__ @@ -230,6 +238,8 @@ with Run(...) as run: Waits until all metadata is processed by Neptune. +Once the call is complete, the data is saved in Neptune. + __Parameters__ | Name | Type | Default | Description | @@ -246,7 +256,7 @@ with Run(...) as run: run.log(...) ... run.wait_for_processing() - run.log(fields={"scores/some_score": some_score_value}) # called once queued Neptune operations have been processed + run.log(fields={"scores/some_score": some_score_value}) # called once submitted data has been processed ``` ## Getting help From 7e16fe0c7c9d9a8255d02892e3f12a238890224b Mon Sep 17 00:00:00 2001 From: Sabine Date: Thu, 29 Aug 2024 12:46:46 +0000 Subject: [PATCH 12/14] Apply suggestions from code review Co-authored-by: Edyta <142720610+szaganek@users.noreply.github.com> --- README.md | 6 +++--- src/neptune_scale/__init__.py | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 99c3dcdf..18e9542d 100644 --- a/README.md +++ b/README.md @@ -68,10 +68,10 @@ __Parameters__ | Name | Type | Default | Description | |------------------|------------------|---------|---------------------------------------------------------------------------| -| `family` | `str` | - | Identifies related runs. All runs of the same lineage must have the same `family` value, i.e forking is only possible within the same family. Max length: 128 characters. | +| `family` | `str` | - | Identifies related runs. All runs of the same lineage must have the same `family` value. That is, forking is only possible within the same family. Max length: 128 characters. | | `run_id` | `str` | - | Identifier of the run. Must be unique within the project. Max length: 128 characters. | | `project` | `str`, optional | `None` | Name of a project in the form `workspace-name/project-name`. If `None`, the value of the `NEPTUNE_PROJECT` environment variable is used. | -| `api_token` | `str`, optional | `None` | Your Neptune API token (or a service account's API token). If `None`, the value of the `NEPTUNE_API_TOKEN` environment variable is used. To keep your token secure, avoid placing it in source code. Instead, save it as an environment variable. | +| `api_token` | `str`, optional | `None` | Your Neptune API token or a service account's API token. If `None`, the value of the `NEPTUNE_API_TOKEN` environment variable is used. To keep your token secure, don't place it in source code. Instead, save it as an environment variable. | | `resume` | `bool`, optional | `False` | If `False` (default), creates a new run. To continue an existing run, set to `True` and pass the ID of an existing run to the `run_id` argument. To fork a run, use `from_run_id` and `from_step` instead. | | `mode` | `Literal`, `"async"` or `"disabled"` | `"async"` | Mode of operation. If set to `"disabled"`, the run doesn't log any metadata. | | `as_experiment` | `str`, optional | `None` | Name of the experiment to associate the run with. Learn more about [experiments](https://docs-beta.neptune.ai/concepts) in the Neptune documentation. | @@ -133,7 +133,7 @@ with Run( Waits for all locally queued data to be processed by Neptune (see [`wait_for_processing()`](#wait_for_processing)) and closes the run. -This is a blocking operation. You should call the function at the end of your script, after your model training is completed. +This is a blocking operation. Call the function at the end of your script, after your model training is completed. __Examples__ diff --git a/src/neptune_scale/__init__.py b/src/neptune_scale/__init__.py index c5419fbe..2076e5ff 100644 --- a/src/neptune_scale/__init__.py +++ b/src/neptune_scale/__init__.py @@ -103,7 +103,7 @@ def __init__( Initializes a Neptune run that logs model-building metadata. Args: - family (str): Identifies related runs. All runs of the same lineage must have the same `family` value, i.e. + family (str): Identifies related runs. All runs of the same lineage must have the same `family` value, that is, forking is only possible within the same family. Max length: 128 characters. run_id (str): Identifier of the run. Must be unique within the project. Max length: 128 characters. project (str): Name of the project where the metadata is logged, in the form `workspace-name/project-name`. From bcaefcfa322097fb14a9a5c274a5341a4146ebbc Mon Sep 17 00:00:00 2001 From: Sabine Date: Thu, 29 Aug 2024 16:27:38 +0000 Subject: [PATCH 13/14] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 18e9542d..ca9855a3 100644 --- a/README.md +++ b/README.md @@ -74,7 +74,7 @@ __Parameters__ | `api_token` | `str`, optional | `None` | Your Neptune API token or a service account's API token. If `None`, the value of the `NEPTUNE_API_TOKEN` environment variable is used. To keep your token secure, don't place it in source code. Instead, save it as an environment variable. | | `resume` | `bool`, optional | `False` | If `False` (default), creates a new run. To continue an existing run, set to `True` and pass the ID of an existing run to the `run_id` argument. To fork a run, use `from_run_id` and `from_step` instead. | | `mode` | `Literal`, `"async"` or `"disabled"` | `"async"` | Mode of operation. If set to `"disabled"`, the run doesn't log any metadata. | -| `as_experiment` | `str`, optional | `None` | Name of the experiment to associate the run with. Learn more about [experiments](https://docs-beta.neptune.ai/concepts) in the Neptune documentation. | +| `as_experiment` | `str`, optional | `None` | Name of the experiment to associate the run with. Learn more about [experiments](https://docs-beta.neptune.ai/experiments) in the Neptune documentation. | | `creation_time` | `datetime`, optional | `None` | Custom creation time of the run. | | `from_run_id` | `str`, optional | `None` | If forking off an existing run, ID of the run to fork from. | | `from_step` | `int`, optional | `None` | If forking off an existing run, step number to fork from. | From 180e962484330bed30103767e867e7e980d3b625 Mon Sep 17 00:00:00 2001 From: Sabine Date: Tue, 3 Sep 2024 17:05:24 +0300 Subject: [PATCH 14/14] Apply suggestions from code review Co-authored-by: Edyta <142720610+szaganek@users.noreply.github.com> --- README.md | 12 +++++++----- src/neptune_scale/__init__.py | 2 +- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index ca9855a3..f9ca54c4 100644 --- a/README.md +++ b/README.md @@ -3,6 +3,8 @@ > [!NOTE] > This package only works with the `3.0` version of neptune.ai called Neptune Scale, which is in beta. > +> It's supported on Linux and MacOS. +> > You can't use the Scale client with the stable Neptune `2.x` versions currently available to SaaS and self-hosting customers. For the Python client corresponding to Neptune `2.x`, see https://github.com/neptune-ai/neptune-client. **What is Neptune?** @@ -18,7 +20,7 @@ Neptune Scale supports forked experiments, with built-in mechanics for retaining ## Installation ```bash -pip install neptune-client-scale +pip install neptune-scale ``` ## Example usage @@ -79,7 +81,7 @@ __Parameters__ | `from_run_id` | `str`, optional | `None` | If forking off an existing run, ID of the run to fork from. | | `from_step` | `int`, optional | `None` | If forking off an existing run, step number to fork from. | | `max_queue_size` | `int`, optional | 1M | Maximum number of operations queued for processing. 1 000 000 by default. You should raise this value if you see the `on_queue_full_callback` function being called. | -| `on_queue_full_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when the queue is full. The function should take two arguments: (1) Exception that made the queue full. (2) Optional timestamp: When the exception was last raised. | +| `on_queue_full_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when the queue is full. The function must take as an argument the exception that made the queue full and, as an optional argument, a timestamp of when the exception was last raised. | | `on_network_error_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when a network error occurs. | | `on_error_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | The default callback function triggered when an unrecoverable error occurs. Applies if an error wasn't caught by other callbacks. In this callback you can choose to perform your cleanup operations and close the training script. | | `on_warning_callback` | `Callable[[BaseException, Optional[float]], None]`, optional | `None` | Callback function triggered when a warning occurs. | @@ -172,9 +174,9 @@ __Parameters__ |---------------|----------------------------------------------------|---------|---------------------------------------------------------------------------| | `step` | `Union[float, int]`, optional | `None` | Index of the log entry. Must be increasing. If not specified, the `log()` call increments the step starting from the highest already logged value. **Tip:** Using float rather than int values can be useful, for example, when logging substeps in a batch. | | `timestamp` | `datetime`, optional | `None` | Time of logging the metadata. | -| `fields` | `Dict[str, Union[float, bool, int, str, datetime, list, set]]`, optional | `None` | Dictionary of configs or other values to log. Independent of the step value. Available types: float, integer, Boolean, string, and datetime. To log multiple values at once, pass multiple dictionaries. | -| `metrics` | `Dict[str, float]`, optional | `None` | Dictionary of metrics to log. Each metric value is associated with a step. To log multiple metrics at once, pass multiple dictionaries. | -| `add_tags` | `Dict[str, Union[List[str], Set[str]]]`, optional | `None` | Dictionary of tags to add to the run, as a list of strings. Independent of the step value. | +| `fields` | `Dict[str, Union[float, bool, int, str, datetime, list, set]]`, optional | `None` | Dictionary of configs or other values to log. Available types: float, integer, Boolean, string, and datetime. | +| `metrics` | `Dict[str, float]`, optional | `None` | Dictionary of metrics to log. Each metric value is associated with a step. To log multiple metrics at once, pass multiple key-value pairs. | +| `add_tags` | `Dict[str, Union[List[str], Set[str]]]`, optional | `None` | Dictionary of tags to add to the run, as a list of strings. | | `remove_tags` | `Dict[str, Union[List[str], Set[str]]]`, optional | `None` | Dictionary of tags to remove from the run, as a list of strings. Independent of the step value. | __Examples__ diff --git a/src/neptune_scale/__init__.py b/src/neptune_scale/__init__.py index 2076e5ff..19ea0b96 100644 --- a/src/neptune_scale/__init__.py +++ b/src/neptune_scale/__init__.py @@ -120,7 +120,7 @@ def __init__( from_step (int): If forking off an existing run, step number to fork from. max_queue_size (int): Maximum number of operations allowed in the queue. on_queue_full_callback (Callable[[BaseException, Optional[float]], None]): Callback function triggered when - the queue is full. The function should take two arguments: + the queue is full. The function takes two arguments: - Exception that made the queue full. - (Optional) Timestamp of the last time the exception was raised. on_network_error_callback: Callback function triggered when a network error occurs.