diff --git a/examples/annotation_import/pdf.ipynb b/examples/annotation_import/pdf.ipynb index 8e037f6a2..bcdd0ab69 100644 --- a/examples/annotation_import/pdf.ipynb +++ b/examples/annotation_import/pdf.ipynb @@ -282,7 +282,7 @@ }, { "metadata": {}, - "source": "client.enable_experimental = True\ntask = lb.DataRow.export(client=client, global_keys=[global_key])\ntask.wait_till_done()\nstream = task.get_buffered_stream()\n\ntext_layer = \"\"\nfor output in stream:\n output_json = json.loads(output.json)\n text_layer = output_json[\"media_attributes\"][\"text_layer_url\"]\nprint(text_layer)", + "source": "client.enable_experimental = True\ntask = lb.DataRow.export(client=client, global_keys=[global_key])\ntask.wait_till_done()\nstream = task.get_buffered_stream()\n\ntext_layer = \"\"\nfor output in stream:\n output_json = output.json\n text_layer = output_json[\"media_attributes\"][\"text_layer_url\"]\nprint(text_layer)", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/basics/basics.ipynb b/examples/basics/basics.ipynb index 3b7796603..268b1a5d6 100644 --- a/examples/basics/basics.ipynb +++ b/examples/basics/basics.ipynb @@ -1,18 +1,16 @@ { - "nbformat": 4, - "nbformat_minor": 5, - "metadata": {}, "cells": [ { + "cell_type": "markdown", "metadata": {}, "source": [ - "", - " ", + "\n", + " \n", "\n" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "\n", @@ -24,162 +22,253 @@ "\n", "" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "# Basic project/datasets overview\n", "\n", "This notebook is used to go over the basic of the Python SDK, such as what a db object is, and how to interact with it. \n", "\n" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "%pip install \"labelbox[data]\"", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "%pip install \"labelbox[data]\"" + ] }, { - "metadata": {}, - "source": "import labelbox as lb", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "import labelbox as lb" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## API key and client\n", "Provide a valid API key below in order to properly connect to the Labelbox Client." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "# Add your API key\nAPI_KEY = None\n# To get your API key go to: Workspace settings -> API -> Create API Key\nclient = lb.Client(api_key=API_KEY)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "# Add your API key\n", + "API_KEY = None\n", + "# To get your API key go to: Workspace settings -> API -> Create API Key\n", + "client = lb.Client(api_key=API_KEY)" + ] }, { - "metadata": {}, - "source": "# For the purpose of this demo get a single project/dataset id from your organization\n\n# Get a single Project id\n# get_projects returns a PaginatedCollection object, which is iterable.\nproject = next(client.get_projects())\nproject_id = project.uid\nproject_name = project.name\nprint(\"Project ID: \", project_id)\nprint(\"Project Name:\", project_name)\n\nprint(\"-\" * 40)\n\n# Get a single dataset id\n# get_datasets returns a PaginatedCollection object, which is iterable.\ndataset = next(client.get_datasets())\ndataset_id = dataset.uid\ndataset_name = dataset.name\nprint(\"Dataset ID: \", dataset_id)\nprint(\"Dataset Name:\", dataset_name)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "# For the purpose of this demo get a single project/dataset id from your organization\n", + "\n", + "# Get a single Project id\n", + "# get_projects returns a PaginatedCollection object, which is iterable.\n", + "project = next(client.get_projects())\n", + "project_id = project.uid\n", + "project_name = project.name\n", + "print(\"Project ID: \", project_id)\n", + "print(\"Project Name:\", project_name)\n", + "\n", + "print(\"-\" * 40)\n", + "\n", + "# Get a single dataset id\n", + "# get_datasets returns a PaginatedCollection object, which is iterable.\n", + "dataset = next(client.get_datasets())\n", + "dataset_id = dataset.uid\n", + "dataset_name = dataset.name\n", + "print(\"Dataset ID: \", dataset_id)\n", + "print(\"Dataset Name:\", dataset_name)" + ] }, { - "metadata": {}, - "source": "# Fetch the project and dataset by using the IDs fetched in the previous cell\nproject = client.get_project(project_id)\ndataset = client.get_dataset(dataset_id)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "# Fetch the project and dataset by using the IDs fetched in the previous cell\n", + "project = client.get_project(project_id)\n", + "dataset = client.get_dataset(dataset_id)" + ] }, { - "metadata": {}, - "source": "print(\"Project: \", project)\nprint(\"Dataset: \", dataset)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "print(\"Project: \", project)\n", + "print(\"Dataset: \", dataset)" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Fields\n", "* All db objects have fields (look at the source code to see them https://github.com/Labelbox/labelbox-python/blob/develop/labelbox/schema/project.py)\n", "* These fields are attributes of the object" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "print(project.name)\nprint(dataset.name)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "print(project.name)\n", + "print(dataset.name)" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "* Fields can be updated. This will be reflected server side (you will see it in labelbox) " - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "project.update(description=\"new description field\")\nprint(project.description)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "project.update(description=\"new description field\")\n", + "print(project.description)" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Pagination\n", "* Queries that return a list of database objects are return as a PaginatedCollection\n", "* Limits the data that is being returned for better performance" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "labels_paginated_collection = project.labels()\nprint(\"Type of collection: \", type(labels_paginated_collection))\n\n# A paginated collection can be parsed by using list()\n# list(paginated...) should be avoided for queries that could return more than a dozen results\nprint(\"Number of labels :\", len(list(labels_paginated_collection)))", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "labels_paginated_collection = project.labels()\n", + "print(\"Type of collection: \", type(labels_paginated_collection))\n", + "\n", + "# A paginated collection can be parsed by using list()\n", + "# list(paginated...) should be avoided for queries that could return more than a dozen results\n", + "print(\"Number of labels :\", len(list(labels_paginated_collection)))" + ] }, { - "metadata": {}, - "source": "# Note that if you selected a `project_id` without any labels this will raise `StopIteration`\n# Iterate over the paginated collection\ntry:\n single_label = next(project.labels())\n print(single_label)\nexcept StopIteration:\n print(\"Project has no labels !\")", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "# Note that if you selected a `project_id` without any labels this will raise `StopIteration`\n", + "# Iterate over the paginated collection\n", + "try:\n", + " single_label = next(project.labels())\n", + " print(single_label)\n", + "except StopIteration:\n", + " print(\"Project has no labels !\")" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Query parameters\n", "* Query with the following conventions:\n", " * `DbObject.Field`" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "datasets = client.get_datasets(where=lb.Dataset.name == dataset_name)\n\nprojects = client.get_projects(\n where=((lb.Project.name == project_name) &\n (lb.Project.description == \"new description field\")))\n\n# The above two queries return PaginatedCollections because the filter parameters aren't guaranteed to be unique.\n# So even if there is one element returned it is in a paginatedCollection.\nprint(projects)\nprint(next(projects, None))\nprint(next(projects, None))\nprint(next(projects, None))\n# We can see there is only one.", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "datasets = client.get_datasets(where=lb.Dataset.name == dataset_name)\n", + "\n", + "projects = client.get_projects(\n", + " where=((lb.Project.name == project_name) &\n", + " (lb.Project.description == \"new description field\")))\n", + "\n", + "# The above two queries return PaginatedCollections because the filter parameters aren't guaranteed to be unique.\n", + "# So even if there is one element returned it is in a paginatedCollection.\n", + "print(projects)\n", + "print(next(projects, None))\n", + "print(next(projects, None))\n", + "print(next(projects, None))\n", + "# We can see there is only one." + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Querying Limitations\n", "* The DbObject used for the query must be the same as the DbObject returned by the querying function. \n", "* The below query is not valid since get_project returns a project not a dataset\n", "> `>>> projects = client.get_projects(where = lb.Dataset.name == \"dataset_name\")`\n" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "# Relationships between projects and batches/datasets\n", "\n" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "sample_project_batches = project.batches()\n\nlist(sample_project_batches)\n\nfor b in sample_project_batches:\n print(f\" Name of project : {b.project().name}\")\n print(f\" Name of batches in project: {b.name}\")", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "sample_project_batches = project.batches()\n", + "\n", + "list(sample_project_batches)\n", + "\n", + "for b in sample_project_batches:\n", + " print(f\" Name of project : {b.project().name}\")\n", + " print(f\" Name of batches in project: {b.name}\")" + ] } - ] -} \ No newline at end of file + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/basics/batches.ipynb b/examples/basics/batches.ipynb index 5b8b4080c..870dcbb23 100644 --- a/examples/basics/batches.ipynb +++ b/examples/basics/batches.ipynb @@ -30,14 +30,15 @@ { "metadata": {}, "source": [ - "# Batches\n", - "https://docs.labelbox.com/docs/batches" + "# Batches" ], "cell_type": "markdown" }, { "metadata": {}, "source": [ + "This notebook covers the basics of batches:\n", + "\n", "* A batch is collection of data rows.\n", "* A data row cannot be part of more than one batch in a given project.\n", "* Batches work for all data types, but there can only be one data type per project.\n", @@ -50,7 +51,14 @@ }, { "metadata": {}, - "source": "%pip install -q \"labelbox[data]\"", + "source": [ + "## Set up" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": "%pip install -q --upgrade \"labelbox[data]\"", "cell_type": "code", "outputs": [], "execution_count": null @@ -66,13 +74,13 @@ "metadata": {}, "source": [ "## API key and client\n", - "Provide a valid API key below in order to properly connect to the Labelbox Client." + "Provide a valid API key below to connect to the Labelbox client properly. For more information, please review the [Create API key](https://docs.labelbox.com/reference/create-api-key) guide." ], "cell_type": "markdown" }, { "metadata": {}, - "source": "# Add your API key\nAPI_KEY = None\n# To get your API key go to: Workspace settings -> API -> Create API Key\nclient = lb.Client(api_key=API_KEY)", + "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", "cell_type": "code", "outputs": [], "execution_count": null @@ -121,7 +129,7 @@ }, { "metadata": {}, - "source": "client.enable_experimental = True\n\nexport_task = dataset.export()\nexport_task.wait_till_done()\n\ndata_rows = []\n\n\ndef json_stream_handler(output: lb.BufferedJsonConverterOutput):\n data_row = json.loads(output.json)\n data_rows.append(data_row)\n\n\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n export_json = export_task.get_buffered_stream(\n stream_type=lb.StreamType.RESULT).start(\n stream_handler=json_stream_handler)", + "source": "export_task = dataset.export()\nexport_task.wait_till_done()\n\ndata_rows = []\n\n\ndef json_stream_handler(output: lb.BufferedJsonConverterOutput):\n data_row = output.json\n data_rows.append(data_row)\n\n\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n export_json = export_task.get_buffered_stream(\n stream_type=lb.StreamType.RESULT).start(\n stream_handler=json_stream_handler)", "cell_type": "code", "outputs": [], "execution_count": null @@ -247,14 +255,14 @@ }, { "metadata": {}, - "source": "client.enable_experimental = True\n\nexport_params = {\n \"attachments\": True,\n \"metadata_fields\": True,\n \"data_row_details\": True,\n \"project_details\": True,\n \"performance_details\": True,\n \"batch_ids\": [\n batch.uid\n ], # Include batch ids if you only want to export specific batches, otherwise,\n # you can export all the data without using this parameter\n}\nfilters = {}\n\n# A task is returned, this provides additional information about the status of your task, such as\n# any errors encountered\nexport_task = project.export(params=export_params, filters=filters)\nexport_task.wait_till_done()", + "source": "export_params = {\n \"attachments\": True,\n \"metadata_fields\": True,\n \"data_row_details\": True,\n \"project_details\": True,\n \"performance_details\": True,\n \"batch_ids\": [\n batch.uid\n ], # Include batch ids if you only want to export specific batches, otherwise,\n # you can export all the data without using this parameter\n}\nfilters = {}\n\n# A task is returned, this provides additional information about the status of your task, such as\n# any errors encountered\nexport_task = project.export(params=export_params, filters=filters)\nexport_task.wait_till_done()", "cell_type": "code", "outputs": [], "execution_count": null }, { "metadata": {}, - "source": "data_rows = []\n\n\ndef json_stream_handler(output: lb.BufferedJsonConverterOutput):\n data_row = json.loads(output.json)\n data_rows.append(data_row)\n\n\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n export_json = export_task.get_buffered_stream(\n stream_type=lb.StreamType.RESULT).start(\n stream_handler=json_stream_handler)", + "source": "data_rows = []\n\n\ndef json_stream_handler(output: lb.BufferedJsonConverterOutput):\n data_row = output.json\n data_rows.append(data_row)\n\n\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n export_json = export_task.get_buffered_stream(\n stream_type=lb.StreamType.RESULT).start(\n stream_handler=json_stream_handler)", "cell_type": "code", "outputs": [], "execution_count": null @@ -283,14 +291,14 @@ { "metadata": {}, "source": [ - "## Clean up \n", - "Uncomment and run the cell below to optionally delete the batch, dataset, and/or project created in this demo." + "## Clean up\n", + "Uncomment and run the cell below to optionally delete Labelbox objects created." ], "cell_type": "markdown" }, { "metadata": {}, - "source": "# Delete Batch\n# batch.delete()\n\n# Delete Project\n# project.delete()\n\n# Delete DataSet\n# dataset.delete()", + "source": "# batch.delete()\n# project.delete()\n# dataset.delete()", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/basics/custom_embeddings.ipynb b/examples/basics/custom_embeddings.ipynb index 7eaf030bc..4c483ba74 100644 --- a/examples/basics/custom_embeddings.ipynb +++ b/examples/basics/custom_embeddings.ipynb @@ -45,7 +45,7 @@ }, { "metadata": {}, - "source": "%pip3 install -q \"labelbox[data]\"", + "source": "%pip install -q --upgrade \"labelbox[data]\"", "cell_type": "code", "outputs": [], "execution_count": null @@ -102,7 +102,7 @@ }, { "metadata": {}, - "source": "data_rows = []\n\n\ndef json_stream_handler(output: lb.BufferedJsonConverterOutput):\n data_row = json.loads(output.json)\n data_rows.append(data_row)\n\n\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n export_json = export_task.get_buffered_stream(\n stream_type=lb.StreamType.RESULT).start(\n stream_handler=json_stream_handler)", + "source": "data_rows = []\n\n\ndef json_stream_handler(output: lb.BufferedJsonConverterOutput):\n data_row = output.json\n data_rows.append(data_row)\n\n\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n export_json = export_task.get_buffered_stream(\n stream_type=lb.StreamType.RESULT).start(\n stream_handler=json_stream_handler)", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/basics/data_rows.ipynb b/examples/basics/data_rows.ipynb index a6c87e691..f17e6fa65 100644 --- a/examples/basics/data_rows.ipynb +++ b/examples/basics/data_rows.ipynb @@ -113,7 +113,7 @@ }, { "metadata": {}, - "source": "data_rows = []\n\n\ndef json_stream_handler(output: lb.BufferedJsonConverterOutput):\n data_row = json.loads(output.json)\n data_rows.append(data_row)\n\n\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n export_json = export_task.get_buffered_stream(\n stream_type=lb.StreamType.RESULT).start(\n stream_handler=json_stream_handler)", + "source": "data_rows = []\n\n\ndef json_stream_handler(output: lb.BufferedJsonConverterOutput):\n data_row = output.json\n data_rows.append(data_row)\n\n\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n export_json = export_task.get_buffered_stream(\n stream_type=lb.StreamType.RESULT).start(\n stream_handler=json_stream_handler)", "cell_type": "code", "outputs": [], "execution_count": null @@ -236,7 +236,7 @@ }, { "metadata": {}, - "source": "# Fetch a data row from the first dataset example\nts = dataset.export()\nts.wait_till_done()\nDATA_ROW_ID = [json.loads(output.json) for output in ts.get_buffered_stream()\n ][0][\"data_row\"][\"id\"]\nGLOBAL_KEY = [json.loads(output.json) for output in ts.get_buffered_stream()\n ][0][\"data_row\"][\"global_key\"]\n\nprint(f\"Pick either a data row id : {DATA_ROW_ID} or global key: {GLOBAL_KEY}\")", + "source": "# Fetch a data row from the first dataset example\nts = dataset.export()\nts.wait_till_done()\nDATA_ROW_ID = [output.json for output in ts.get_buffered_stream()\n ][0][\"data_row\"][\"id\"]\nGLOBAL_KEY = [output.json for output in ts.get_buffered_stream()\n ][0][\"data_row\"][\"global_key\"]\n\nprint(f\"Pick either a data row id : {DATA_ROW_ID} or global key: {GLOBAL_KEY}\")", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/basics/projects.ipynb b/examples/basics/projects.ipynb index c9e5e5813..6bebba732 100644 --- a/examples/basics/projects.ipynb +++ b/examples/basics/projects.ipynb @@ -30,7 +30,8 @@ { "metadata": {}, "source": [ - "# Projects" + "# Projects\n", + "This notebook covers the basics of projects:" ], "cell_type": "markdown" }, @@ -47,7 +48,14 @@ }, { "metadata": {}, - "source": "%pip install -q \"labelbox[data]\"", + "source": [ + "## Set up" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": "%pip install -q --upgrade \"labelbox[data]\"", "cell_type": "code", "outputs": [], "execution_count": null @@ -62,14 +70,14 @@ { "metadata": {}, "source": [ - "# API Key and Client\n", - "Provide a valid api key below in order to properly connect to the Labelbox Client." + "## API key and client\n", + "Provide a valid API key below to connect to the Labelbox client properly. For more information, please review the [Create API key](https://docs.labelbox.com/reference/create-api-key) guide." ], "cell_type": "markdown" }, { "metadata": {}, - "source": "# Add your API key\nAPI_KEY = \"\"\n# To get your API key go to: Workspace settings -> API -> Create API Key\nclient = lb.Client(api_key=API_KEY)", + "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", "cell_type": "code", "outputs": [], "execution_count": null @@ -355,7 +363,8 @@ { "metadata": {}, "source": [ - "### Clean Up" + "## Clean up\n", + "Uncomment and run the cell below to optionally delete Labelbox objects created." ], "cell_type": "markdown" }, diff --git a/examples/exports/composite_mask_export.ipynb b/examples/exports/composite_mask_export.ipynb index 60e2cb0c7..495dc0758 100644 --- a/examples/exports/composite_mask_export.ipynb +++ b/examples/exports/composite_mask_export.ipynb @@ -42,13 +42,13 @@ { "metadata": {}, "source": [ - "## Imports" + "## Set up" ], "cell_type": "markdown" }, { "metadata": {}, - "source": "%pip install -q \"labelbox[data]\"", + "source": "%pip install -q --upgrade \"labelbox[data]\"", "cell_type": "code", "outputs": [], "execution_count": null @@ -63,14 +63,14 @@ { "metadata": {}, "source": [ - "## API Key and Client\n", - "See the developer guide for [creating an API key](https://docs.labelbox.com/reference/create-api-key)." + "## API key and client\n", + "Provide a valid API key below in order to properly connect to the Labelbox client. Please review [Create API key guide](https://docs.labelbox.com/reference/create-api-key) for more information." ], "cell_type": "markdown" }, { "metadata": {}, - "source": "API_KEY = \"\"\nclient = lb.Client(api_key=API_KEY)\nclient.enable_experimental = (\n True ## This is required if using the export() streamable method\n)", + "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", "cell_type": "code", "outputs": [], "execution_count": null @@ -85,7 +85,7 @@ { "metadata": {}, "source": [ - "#### Composite masks\n", + "### Composite masks\n", "**IMPORTANT :** The URL for the ```composite_mask``` from exports older than 30 days will no longer be accessible. To obtain a functional URL after this period, please generate a new export for the mask.\n", "1. A single mask url contains all mask instances from a single label. For videos a composite mask contains all mask instances for the frame in each label. \n", "2. The export and mask URL adhere to the following convention:\n", @@ -157,15 +157,7 @@ { "metadata": {}, "source": [ - "---" - ], - "cell_type": "markdown" - }, - { - "metadata": {}, - "source": [ - "#### Single mask instance:\n", - "We are planning on removing single mask instances, but for now they will be displayed adjecent to the composite mask.\n", + "### Single mask instance:\n", "1. A single mask instance and mask url is generated for each individual annotation for each label.\n", "2. The export and mask URL adhere to the following convention: \n", "```json\n", @@ -183,7 +175,7 @@ { "metadata": {}, "source": [ - "## Create an export from an Image project with mask annotations\n", + "## Create an export from a project with mask annotations\n", "To better showcase composite masks, make sure you have different mask tools and mask annotations in your project" ], "cell_type": "markdown" @@ -211,7 +203,7 @@ }, { "metadata": {}, - "source": "stream = export_task.get_buffered_stream()\n\nmask_tool_rgb_mapping = {}\n\nfor output in stream:\n # Parse the JSON string from the output\n output_json = json.loads(output.json)\n\n # Get the labels for the specified project ID or an empty list if the project ID is not found\n project_labels = (output_json[\"projects\"].get(PROJECT_ID,\n {}).get(\"labels\", []))\n\n # Iterate through each label\n for label in project_labels:\n # Get the list of annotations (objects) for the label\n annotations = label[\"annotations\"].get(\"objects\", [])\n\n # Iterate through each annotation\n for annotation in annotations:\n # Check if the annotation is of type \"ImageSegmentationMask\"\n if annotation.get(\"annotation_kind\") == \"ImageSegmentationMask\":\n # Add the color RGB information to the mapping dictionary\n mask_tool_rgb_mapping.setdefault(annotation[\"name\"], []).append(\n annotation[\"composite_mask\"][\"color_rgb\"])\n\nprint(mask_tool_rgb_mapping)", + "source": "stream = export_task.get_buffered_stream()\n\nmask_tool_rgb_mapping = {}\n\nfor output in stream:\n # Parse the JSON string from the output\n output_json = output.json\n\n # Get the labels for the specified project ID or an empty list if the project ID is not found\n project_labels = (output_json[\"projects\"].get(PROJECT_ID,\n {}).get(\"labels\", []))\n\n # Iterate through each label\n for label in project_labels:\n # Get the list of annotations (objects) for the label\n annotations = label[\"annotations\"].get(\"objects\", [])\n\n # Iterate through each annotation\n for annotation in annotations:\n # Check if the annotation is of type \"ImageSegmentationMask\"\n if annotation.get(\"annotation_kind\") == \"ImageSegmentationMask\":\n # Add the color RGB information to the mapping dictionary\n mask_tool_rgb_mapping.setdefault(annotation[\"name\"], []).append(\n annotation[\"composite_mask\"][\"color_rgb\"])\n\nprint(mask_tool_rgb_mapping)", "cell_type": "code", "outputs": [], "execution_count": null @@ -246,7 +238,7 @@ }, { "metadata": {}, - "source": "tools_frames_color = {}\nstream = export_task_video.get_buffered_stream()\n\n# Iterate over each output in the stream\nfor output in stream:\n output_json = json.loads(output.json)\n\n # Iterate over the labels in the specific project\n for dr in output_json[\"projects\"][VIDEO_PROJECT_ID][\"labels\"]:\n frames_data = dr[\"annotations\"][\"frames\"]\n\n # Iterate over each frame in the frames data\n for frame_key, frame_value in frames_data.items():\n\n # Iterate over each annotation in the frame\n for annotation_key, annotation_value in frame_value.items():\n if \"objects\" in annotation_key and annotation_value.values():\n\n # Iterate over each object in the annotation\n for object_key, object_value in annotation_value.items():\n if (object_value[\"annotation_kind\"] ==\n \"VideoSegmentationMask\"):\n # Update tools_frames_color with object information\n tools_frames_color.setdefault(\n object_value[\"name\"], []).append({\n frame_key:\n object_value[\"composite_mask\"]\n [\"color_rgb\"]\n })\n\nprint(tools_frames_color)", + "source": "tools_frames_color = {}\nstream = export_task_video.get_buffered_stream()\n\n# Iterate over each output in the stream\nfor output in stream:\n output_json = output.json\n\n # Iterate over the labels in the specific project\n for dr in output_json[\"projects\"][VIDEO_PROJECT_ID][\"labels\"]:\n frames_data = dr[\"annotations\"][\"frames\"]\n\n # Iterate over each frame in the frames data\n for frame_key, frame_value in frames_data.items():\n\n # Iterate over each annotation in the frame\n for annotation_key, annotation_value in frame_value.items():\n if \"objects\" in annotation_key and annotation_value.values():\n\n # Iterate over each object in the annotation\n for object_key, object_value in annotation_value.items():\n if (object_value[\"annotation_kind\"] ==\n \"VideoSegmentationMask\"):\n # Update tools_frames_color with object information\n tools_frames_color.setdefault(\n object_value[\"name\"], []).append({\n frame_key:\n object_value[\"composite_mask\"]\n [\"color_rgb\"]\n })\n\nprint(tools_frames_color)", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/integrations/langchain/langchain.ipynb b/examples/integrations/langchain/langchain.ipynb index f6653d001..90c840844 100644 --- a/examples/integrations/langchain/langchain.ipynb +++ b/examples/integrations/langchain/langchain.ipynb @@ -1,18 +1,16 @@ { - "nbformat": 4, - "nbformat_minor": 0, - "metadata": {}, "cells": [ { + "cell_type": "markdown", "metadata": {}, "source": [ - "", - " ", + "\n", + " \n", "\n" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "\n", @@ -24,153 +22,415 @@ "\n", "" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "# LangChain -> Labelbox\n", "This notebook is used to show an example workflow of getting LangChain traces into Labelbox conversation data format. Please review the [associated written guide](https://labelbox.com/guides/turn-langchain-logs-into-conversational-data-with-labelbox/) for more information." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "%pip install --upgrade --quiet langchain langsmith langchainhub\n%pip install --upgrade --quiet langchain-openai tiktoken pandas duckduckgo-search\n%pip install --upgrade --quiet \"labelbox[data]\"", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "%pip install --upgrade --quiet langchain langsmith langchainhub\n", + "%pip install --upgrade --quiet langchain-openai tiktoken pandas duckduckgo-search\n", + "%pip install --upgrade --quiet \"labelbox[data]\"" + ] }, { - "metadata": {}, - "source": "import labelbox as lb\nfrom uuid import uuid4\nimport os\nimport functools\n\n# LangSmith Imports\nfrom langsmith.client import Client\nfrom langchain import hub\nfrom langchain.agents import AgentExecutor\nfrom langchain.agents.format_scratchpad.openai_tools import (\n format_to_openai_tool_messages,)\nfrom langchain.agents.output_parsers.openai_tools import (\n OpenAIToolsAgentOutputParser,)\nfrom langchain_community.tools import DuckDuckGoSearchResults\nfrom langchain_openai import ChatOpenAI\nfrom langsmith.evaluation import EvaluationResult\nfrom langsmith.schemas import Example, Run, DataType\nfrom langchain.smith import run_on_dataset\nfrom langchain.evaluation import EvaluatorType\nfrom langchain.smith import RunEvalConfig", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "import labelbox as lb\n", + "from uuid import uuid4\n", + "import os\n", + "import functools\n", + "\n", + "# LangSmith Imports\n", + "from langsmith.client import Client\n", + "from langchain import hub\n", + "from langchain.agents import AgentExecutor\n", + "from langchain.agents.format_scratchpad.openai_tools import (\n", + " format_to_openai_tool_messages,)\n", + "from langchain.agents.output_parsers.openai_tools import (\n", + " OpenAIToolsAgentOutputParser,)\n", + "from langchain_community.tools import DuckDuckGoSearchResults\n", + "from langchain_openai import ChatOpenAI\n", + "from langsmith.evaluation import EvaluationResult\n", + "from langsmith.schemas import Example, Run, DataType\n", + "from langchain.smith import run_on_dataset\n", + "from langchain.evaluation import EvaluatorType\n", + "from langchain.smith import RunEvalConfig" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## API Key and Setup\n", "Provide a valid API key below for Labelbox, LangSmith and OpenAI in order for the notebook to work correctly." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "LB_API_KEY = \"\"\nLS_API_KEY = \"\"\nOPENAI_API_KEY = \"\"\n\nunique_id = uuid4().hex[0:8]\nos.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\nos.environ[\"LANGCHAIN_PROJECT\"] = f\"Tracing Walkthrough - {unique_id}\"\nos.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\nos.environ[\"LANGCHAIN_API_KEY\"] = LS_API_KEY\n\nos.environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY\n\nlb_client = lb.Client(LB_API_KEY)\nclient = Client()", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "LB_API_KEY = \"\"\n", + "LS_API_KEY = \"\"\n", + "OPENAI_API_KEY = \"\"\n", + "\n", + "unique_id = uuid4().hex[0:8]\n", + "os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n", + "os.environ[\"LANGCHAIN_PROJECT\"] = f\"Tracing Walkthrough - {unique_id}\"\n", + "os.environ[\"LANGCHAIN_ENDPOINT\"] = \"https://api.smith.langchain.com\"\n", + "os.environ[\"LANGCHAIN_API_KEY\"] = LS_API_KEY\n", + "\n", + "os.environ[\"OPENAI_API_KEY\"] = OPENAI_API_KEY\n", + "\n", + "lb_client = lb.Client(LB_API_KEY)\n", + "client = Client()" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### LangSmith Dataset Name\n", "Create a sample chat data set with an example chat based run" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "dataset_name = f\"Sample Dataset - {str(uuid4())}\"\ndataset = client.create_dataset(\n dataset_name,\n description=\"A sample dataset in LangSmith.\",\n data_type=DataType.chat,\n)\nclient.create_chat_example(\n messages=[\n {\n \"type\": \"ai\",\n \"data\": {\n \"content\": \"hi how are you\"\n }\n },\n {\n \"type\": \"human\",\n \"data\": {\n \"content\": \"Im doing great how about you\"\n }\n },\n ],\n generations={\n \"type\": \"ai\",\n \"data\": {\n \"content\": \"Im doing great\"\n },\n }, # Custom model output\n dataset_id=dataset.id,\n)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "dataset_name = f\"Sample Dataset - {str(uuid4())}\"\n", + "dataset = client.create_dataset(\n", + " dataset_name,\n", + " description=\"A sample dataset in LangSmith.\",\n", + " data_type=DataType.chat,\n", + ")\n", + "client.create_chat_example(\n", + " messages=[\n", + " {\n", + " \"type\": \"ai\",\n", + " \"data\": {\n", + " \"content\": \"hi how are you\"\n", + " }\n", + " },\n", + " {\n", + " \"type\": \"human\",\n", + " \"data\": {\n", + " \"content\": \"Im doing great how about you\"\n", + " }\n", + " },\n", + " ],\n", + " generations={\n", + " \"type\": \"ai\",\n", + " \"data\": {\n", + " \"content\": \"Im doing great\"\n", + " },\n", + " }, # Custom model output\n", + " dataset_id=dataset.id,\n", + ")" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### LangSmith\n", "Below is an example of running a list of raw text evaluation strings and a LangSmith example run with Chat Gpt 3.5. Please review [LangSmith Docs](https://docs.smith.langchain.com/) for more information." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "tools = [\n DuckDuckGoSearchResults(\n name=\"duck_duck_go\"), # General internet search using DuckDuckGo\n]\n\nllm = ChatOpenAI(\n model=\"gpt-3.5-turbo-16k\",\n temperature=0,\n)\n\n\n# Since chains can be stateful (e.g. they can have memory), we provide\n# a way to initialize a new chain for each row in the dataset. This is done\n# by passing in a factory function that returns a new chain for each row.\ndef create_agent(prompt, llm_with_tools):\n runnable_agent = ({\n \"input\":\n lambda x: x[\"input\"],\n \"agent_scratchpad\":\n lambda x: format_to_openai_tool_messages(x[\"intermediate_steps\"]),\n } | prompt | llm_with_tools | OpenAIToolsAgentOutputParser())\n return AgentExecutor(agent=runnable_agent,\n tools=tools,\n handle_parsing_errors=True)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "tools = [\n", + " DuckDuckGoSearchResults(\n", + " name=\"duck_duck_go\"), # General internet search using DuckDuckGo\n", + "]\n", + "\n", + "llm = ChatOpenAI(\n", + " model=\"gpt-3.5-turbo-16k\",\n", + " temperature=0,\n", + ")\n", + "\n", + "\n", + "# Since chains can be stateful (e.g. they can have memory), we provide\n", + "# a way to initialize a new chain for each row in the dataset. This is done\n", + "# by passing in a factory function that returns a new chain for each row.\n", + "def create_agent(prompt, llm_with_tools):\n", + " runnable_agent = ({\n", + " \"input\":\n", + " lambda x: x[\"input\"],\n", + " \"agent_scratchpad\":\n", + " lambda x: format_to_openai_tool_messages(x[\"intermediate_steps\"]),\n", + " } | prompt | llm_with_tools | OpenAIToolsAgentOutputParser())\n", + " return AgentExecutor(agent=runnable_agent,\n", + " tools=tools,\n", + " handle_parsing_errors=True)" + ] }, { - "metadata": {}, - "source": "def max_pred_length(runs, examples):\n predictions = [len(run.outputs[\"output\"]) for run in runs]\n return EvaluationResult(key=\"max_pred_length\", score=max(predictions))", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "def max_pred_length(runs, examples):\n", + " predictions = [len(run.outputs[\"output\"]) for run in runs]\n", + " return EvaluationResult(key=\"max_pred_length\", score=max(predictions))" + ] }, { - "metadata": {}, - "source": "def check_not_idk(run: Run, example: Example):\n \"\"\"Illustration of a custom evaluator.\"\"\"\n agent_response = run.outputs[\"output\"]\n if \"don't know\" in agent_response or \"not sure\" in agent_response:\n score = 0\n else:\n score = 1\n # You can access the dataset labels in example.outputs[key]\n # You can also access the model inputs in run.inputs[key]\n return EvaluationResult(\n key=\"not_uncertain\",\n score=score,\n )", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "def check_not_idk(run: Run, example: Example):\n", + " \"\"\"Illustration of a custom evaluator.\"\"\"\n", + " agent_response = run.outputs[\"output\"]\n", + " if \"don't know\" in agent_response or \"not sure\" in agent_response:\n", + " score = 0\n", + " else:\n", + " score = 1\n", + " # You can access the dataset labels in example.outputs[key]\n", + " # You can also access the model inputs in run.inputs[key]\n", + " return EvaluationResult(\n", + " key=\"not_uncertain\",\n", + " score=score,\n", + " )" + ] }, { - "metadata": {}, - "source": "evaluation_config = RunEvalConfig(\n # Evaluators can either be an evaluator type (e.g., \"qa\", \"criteria\", \"embedding_distance\", etc.) or a configuration for that evaluator\n evaluators=[\n check_not_idk,\n # Measures whether a QA response is \"Correct\", based on a reference answer\n # You can also select via the raw string \"qa\"\n EvaluatorType.QA,\n # Measure the embedding distance between the output and the reference answer\n # Equivalent to: EvalConfig.EmbeddingDistance(embeddings=OpenAIEmbeddings())\n EvaluatorType.EMBEDDING_DISTANCE,\n # Grade whether the output satisfies the stated criteria.\n # You can select a default one such as \"helpfulness\" or provide your own.\n RunEvalConfig.LabeledCriteria(\"helpfulness\"),\n # The LabeledScoreString evaluator outputs a score on a scale from 1-10.\n # You can use default criteria or write our own rubric\n RunEvalConfig.LabeledScoreString(\n {\n \"accuracy\":\n \"\"\"\nScore 1: The answer is completely unrelated to the reference.\nScore 3: The answer has minor relevance but does not align with the reference.\nScore 5: The answer has moderate relevance but contains inaccuracies.\nScore 7: The answer aligns with the reference but has minor errors or omissions.\nScore 10: The answer is completely accurate and aligns perfectly with the reference.\"\"\"\n },\n normalize_by=10,\n ),\n ],\n batch_evaluators=[max_pred_length],\n)\n\nllm_with_tools = llm.bind_tools(tools)\nprompt = hub.pull(\"gabe/labelboxtutorialdemo\"\n ) # Change prompt in LangSmith hub to reflect example run\n\nchain_results = run_on_dataset(\n dataset_name=dataset_name,\n llm_or_chain_factory=functools.partial(create_agent,\n prompt=prompt,\n llm_with_tools=llm_with_tools),\n evaluation=evaluation_config,\n verbose=True,\n client=client,\n project_name=f\"tools-agent-test-5d466cbc-{unique_id}\",\n # Project metadata communicates the experiment parameters,\n # Useful for reviewing the test results\n project_metadata={\n \"env\": \"testing-notebook\",\n \"model\": \"gpt-3.5-turbo\",\n \"prompt\": \"5d466cbc\",\n },\n)\n\n# Sometimes, the agent will error due to parsing issues, incompatible tool inputs, etc.\n# These are logged as warnings here and captured as errors in the tracing UI.", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "evaluation_config = RunEvalConfig(\n", + " # Evaluators can either be an evaluator type (e.g., \"qa\", \"criteria\", \"embedding_distance\", etc.) or a configuration for that evaluator\n", + " evaluators=[\n", + " check_not_idk,\n", + " # Measures whether a QA response is \"Correct\", based on a reference answer\n", + " # You can also select via the raw string \"qa\"\n", + " EvaluatorType.QA,\n", + " # Measure the embedding distance between the output and the reference answer\n", + " # Equivalent to: EvalConfig.EmbeddingDistance(embeddings=OpenAIEmbeddings())\n", + " EvaluatorType.EMBEDDING_DISTANCE,\n", + " # Grade whether the output satisfies the stated criteria.\n", + " # You can select a default one such as \"helpfulness\" or provide your own.\n", + " RunEvalConfig.LabeledCriteria(\"helpfulness\"),\n", + " # The LabeledScoreString evaluator outputs a score on a scale from 1-10.\n", + " # You can use default criteria or write our own rubric\n", + " RunEvalConfig.LabeledScoreString(\n", + " {\n", + " \"accuracy\":\n", + " \"\"\"\n", + "Score 1: The answer is completely unrelated to the reference.\n", + "Score 3: The answer has minor relevance but does not align with the reference.\n", + "Score 5: The answer has moderate relevance but contains inaccuracies.\n", + "Score 7: The answer aligns with the reference but has minor errors or omissions.\n", + "Score 10: The answer is completely accurate and aligns perfectly with the reference.\"\"\"\n", + " },\n", + " normalize_by=10,\n", + " ),\n", + " ],\n", + " batch_evaluators=[max_pred_length],\n", + ")\n", + "\n", + "llm_with_tools = llm.bind_tools(tools)\n", + "prompt = hub.pull(\"gabe/labelboxtutorialdemo\"\n", + " ) # Change prompt in LangSmith hub to reflect example run\n", + "\n", + "chain_results = run_on_dataset(\n", + " dataset_name=dataset_name,\n", + " llm_or_chain_factory=functools.partial(create_agent,\n", + " prompt=prompt,\n", + " llm_with_tools=llm_with_tools),\n", + " evaluation=evaluation_config,\n", + " verbose=True,\n", + " client=client,\n", + " project_name=f\"tools-agent-test-5d466cbc-{unique_id}\",\n", + " # Project metadata communicates the experiment parameters,\n", + " # Useful for reviewing the test results\n", + " project_metadata={\n", + " \"env\": \"testing-notebook\",\n", + " \"model\": \"gpt-3.5-turbo\",\n", + " \"prompt\": \"5d466cbc\",\n", + " },\n", + ")\n", + "\n", + "# Sometimes, the agent will error due to parsing issues, incompatible tool inputs, etc.\n", + "# These are logged as warnings here and captured as errors in the tracing UI." + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Labelbox\n", "Below converts the results of the above LangSmith run to Labelbox conversation text. Please review [Labelbox conversation data docs](https://docs.labelbox.com/docs/llm-human-preference) for more information." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "def import_conversational(\n chain_results: dict[str:str],\n user_id_dict: dict[str:dict[str:str]]) -> dict[str:str]:\n \"\"\"Converts LangSmith chain_results from model invocation to Labelbox conversation data for model response comparison. Output is based on popular model response and custom model response towards prompts.\n\n Args:\n chain_results(dict[str:str]): Results from LangSmith model invocation against example dataset runs.\n user_id_dict(dict[str:dict[str:str]]): Dictionary mapping of LangSmith example run type to Labelbox chat names and alignment.\n\n Returns:\n dict[str:str]: Labelbox conversation text format\n \"\"\"\n lb_conversations = []\n for key, conversational in chain_results[\"results\"].items():\n lb_conversation = {\n \"row_data\": {\n \"type\": \"application/vnd.labelbox.conversational\",\n \"version\": 1,\n \"messages\": [],\n \"modelOutputs\": [],\n },\n \"global_key\": key,\n \"media_type\": \"CONVERSATIONAL\",\n }\n if \"input\" in conversational[\"output\"]:\n for i, input in enumerate(conversational[\"output\"][\"input\"]):\n lb_conversation[\"row_data\"][\"messages\"].append({\n \"content\": input[\"data\"][\"content\"],\n \"timestampUsec\": i + 1,\n \"user\": {\n \"userId\": user_id_dict[input[\"type\"]][\"id\"],\n \"name\": input[\"type\"],\n },\n \"canLabel\": True,\n \"align\": user_id_dict[input[\"type\"]][\"align\"],\n \"messageId\": str(uuid4()),\n })\n\n # Custom model output\n if \"reference\" in conversational:\n reference = conversational[\"reference\"][\"output\"]\n lb_conversation[\"row_data\"][\"modelOutputs\"].append({\n \"title\": \"Custom Model Response\",\n \"content\": reference[\"data\"][\"content\"],\n \"modelConfigName\": \"Custom Model - Example Config\",\n })\n\n # Popular model output\n if \"output\" in conversational[\"output\"]:\n output = conversational[\"output\"][\"output\"]\n lb_conversation[\"row_data\"][\"modelOutputs\"].append({\n \"title\": \"Popular LLM Response\",\n \"content\": output,\n \"modelConfigName\": \"GPT-3.5 - Example Config\",\n })\n\n lb_conversations.append(lb_conversation)\n return lb_conversations", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "def import_conversational(\n", + " chain_results: dict[str:str],\n", + " user_id_dict: dict[str:dict[str:str]]) -> dict[str:str]:\n", + " \"\"\"Converts LangSmith chain_results from model invocation to Labelbox conversation data for model response comparison. Output is based on popular model response and custom model response towards prompts.\n", + "\n", + " Args:\n", + " chain_results(dict[str:str]): Results from LangSmith model invocation against example dataset runs.\n", + " user_id_dict(dict[str:dict[str:str]]): Dictionary mapping of LangSmith example run type to Labelbox chat names and alignment.\n", + "\n", + " Returns:\n", + " dict[str:str]: Labelbox conversation text format\n", + " \"\"\"\n", + " lb_conversations = []\n", + " for key, conversational in chain_results[\"results\"].items():\n", + " lb_conversation = {\n", + " \"row_data\": {\n", + " \"type\": \"application/vnd.labelbox.conversational\",\n", + " \"version\": 1,\n", + " \"messages\": [],\n", + " \"modelOutputs\": [],\n", + " },\n", + " \"global_key\": key,\n", + " \"media_type\": \"CONVERSATIONAL\",\n", + " }\n", + " if \"input\" in conversational[\"output\"]:\n", + " for i, input in enumerate(conversational[\"output\"][\"input\"]):\n", + " lb_conversation[\"row_data\"][\"messages\"].append({\n", + " \"content\": input[\"data\"][\"content\"],\n", + " \"timestampUsec\": i + 1,\n", + " \"user\": {\n", + " \"userId\": user_id_dict[input[\"type\"]][\"id\"],\n", + " \"name\": input[\"type\"],\n", + " },\n", + " \"canLabel\": True,\n", + " \"align\": user_id_dict[input[\"type\"]][\"align\"],\n", + " \"messageId\": str(uuid4()),\n", + " })\n", + "\n", + " # Custom model output\n", + " if \"reference\" in conversational:\n", + " reference = conversational[\"reference\"][\"output\"]\n", + " lb_conversation[\"row_data\"][\"modelOutputs\"].append({\n", + " \"title\": \"Custom Model Response\",\n", + " \"content\": reference[\"data\"][\"content\"],\n", + " \"modelConfigName\": \"Custom Model - Example Config\",\n", + " })\n", + "\n", + " # Popular model output\n", + " if \"output\" in conversational[\"output\"]:\n", + " output = conversational[\"output\"][\"output\"]\n", + " lb_conversation[\"row_data\"][\"modelOutputs\"].append({\n", + " \"title\": \"Popular LLM Response\",\n", + " \"content\": output,\n", + " \"modelConfigName\": \"GPT-3.5 - Example Config\",\n", + " })\n", + "\n", + " lb_conversations.append(lb_conversation)\n", + " return lb_conversations" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Create Labelbox Dataset" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "dataset = lb_client.create_dataset(name=\"demo_langchain\")", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "dataset = lb_client.create_dataset(name=\"demo_langchain\")" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Attach Conversation Text to Dataset" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "task = dataset.create_data_rows(\n import_conversational(\n chain_results,\n {\n \"human\": {\n \"id\": \"human\",\n \"align\": \"right\"\n },\n \"ai\": {\n \"id\": \"ai\",\n \"align\": \"left\"\n },\n },\n ))\ntask.wait_till_done()\n\nprint(task.errors)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "task = dataset.create_data_rows(\n", + " import_conversational(\n", + " chain_results,\n", + " {\n", + " \"human\": {\n", + " \"id\": \"human\",\n", + " \"align\": \"right\"\n", + " },\n", + " \"ai\": {\n", + " \"id\": \"ai\",\n", + " \"align\": \"left\"\n", + " },\n", + " },\n", + " ))\n", + "task.wait_till_done()\n", + "\n", + "print(task.errors)" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Cleanup" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "# dataset.delete()", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "# dataset.delete()" + ] } - ] -} \ No newline at end of file + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/integrations/yolo/import_yolo_annotations.ipynb b/examples/integrations/yolo/import_yolo_annotations.ipynb new file mode 100644 index 000000000..59ea6f0d8 --- /dev/null +++ b/examples/integrations/yolo/import_yolo_annotations.ipynb @@ -0,0 +1,189 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + " \n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Import YOLOv8 annotations\n", + "This notebook will provides examples of setting up a Project with annotations generated with YOLOv8. We will be using the [Ultralytics](https://docs.ultralytics.com/) library to generate our annotations. In this guide we will be:\n", + "* Importing a demo image data rows that will be labeled\n", + "* Setting up our ontology that matches our YOLOv8 annotations\n", + "* Importing our data rows and attaching our ontology to a project\n", + "* Running our images through Ultralytics then importing the annotations generated\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set up" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n", + "\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", + "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", + "Note: you may need to restart the kernel to use updated packages.\n" + ] + } + ], + "source": [ + "%pip install -q --upgrade \"labelbox[data]\"\n", + "%pip install -q --upgrade ultralytics" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "WARNING ⚠️ Ultralytics settings reset to default values. This may be due to a possible problem with your settings or a recent ultralytics package update. \n", + "View settings with 'yolo settings' or at '/Users/gabrielunderwood/Library/Application Support/Ultralytics/settings.yaml'\n", + "Update settings with 'yolo settings key=value', i.e. 'yolo settings runs_dir=path/to/dir'. For help see https://docs.ultralytics.com/quickstart/#ultralytics-settings.\n" + ] + } + ], + "source": [ + "import labelbox as lb\n", + "import labelbox.types as lb_types\n", + "import uuid\n", + "import ultralytics\n", + "from ultralytics.engine.model import Model\n", + "from ultralytics.engine.results import Results" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API key and client\n", + "Provide a valid API key below in order to properly connect to the Labelbox client. Please review [Create API key guide](https://docs.labelbox.com/reference/create-api-key) for more information." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "API_KEY = \"eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ1c2VySWQiOiJjbG9vcmRpaGUwMDkyMDcza2Nvcm5jajdnIiwib3JnYW5pemF0aW9uSWQiOiJjbG9vcmRpZ3cwMDkxMDcza2M2cG9oeWFiIiwiYXBpS2V5SWQiOiJjbHhhZnpkcWgwMmZqMDd5MTdramllZW8xIiwic2VjcmV0IjoiZjkxMmZhMzMwOTRmOGMxYzFkN2QzNTY2NTMxMzEyNDMiLCJpYXQiOjE3MTgxMTI3NjksImV4cCI6MjM0OTI2NDc2OX0.4wY66g9LaP5xFHoSeleAgTK75CX7wxKMsaIbX0nI_qc\"\n", + "client = lb.Client(api_key=API_KEY)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example: Import YOLOv8 Annotations\n", + "\n", + "The first few steps of this guide will demonstrating a basic workflow of creating data rows and setting up a project. For a quick complete overview of this process visit our [Quick start](https://docs.labelbox.com/reference/quick-start) guide." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Importing an image data row\n", + "\n", + "We will be using this [image](https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg) to be annotated with YOLOv8. Which has a lot of objects that can be picked up by YOLOv8. Later in this guide we will go into more detail on the exact annotations." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Errors: None\n" + ] + } + ], + "source": [ + "# create data row\n", + "data_row = {\n", + " \"row_data\": \"https://storage.googleapis.com/labelbox-datasets/image_sample_data/2560px-Kitano_Street_Kobe01s5s4110.jpeg\",\n", + " \"global_key\": str(uuid.uuid4()),\n", + " \"media_type\": \"IMAGE\"\n", + "}\n", + "\n", + "# create dataset and import data row\n", + "dataset = client.create_dataset(name=\"YOLOv8 Demo Dataset\")\n", + "task = dataset.create_data_rows([data_row])\n", + "task.wait_till_done()\n", + "\n", + "print(f\"Errors: {task.errors}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Setting up an ontology and a project\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/model_experiments/custom_metrics_demo.ipynb b/examples/model_experiments/custom_metrics_demo.ipynb index ed8516d2a..28a63c011 100644 --- a/examples/model_experiments/custom_metrics_demo.ipynb +++ b/examples/model_experiments/custom_metrics_demo.ipynb @@ -30,8 +30,6 @@ { "metadata": {}, "source": [ - "----\n", - "\n", "# Model Diagnostics - Custom Metrics Demo\n", "\n", "* Measuring model quality is critical to efficiently building models. It is important that the metrics used to measure model quality closely align with the business objectives for the model. Otherwise, slight changes in model quality, as they related to these core objectives, are lost to noise. Custom metrics enables users to measure model quality in terms of their exact business goals. By incorporating custom metrics into workflows, users can:\n", @@ -44,13 +42,13 @@ { "metadata": {}, "source": [ - "## Environment setup" + "## Set up" ], "cell_type": "markdown" }, { "metadata": {}, - "source": "%pip install -q \"labelbox[data]\"", + "source": "%pip install -q --upgrade \"labelbox[data]\"", "cell_type": "code", "outputs": [], "execution_count": null @@ -65,14 +63,14 @@ { "metadata": {}, "source": [ - "## Replace with your API Key\n", - "Guides on [Create an API key](https://docs.labelbox.com/docs/create-an-api-key)" + "## API key and client\n", + "Provide a valid API key below to connect to the Labelbox client properly. For more information, please review the [Create API key](https://docs.labelbox.com/reference/create-api-key) guide." ], "cell_type": "markdown" }, { "metadata": {}, - "source": "API_KEY = \"\"\nclient = lb.Client(API_KEY)", + "source": "API_KEY = None\nclient = lb.Client(API_KEY)", "cell_type": "code", "outputs": [], "execution_count": null @@ -87,7 +85,14 @@ { "metadata": {}, "source": [ - "### Classification: Radio (single-choice)" + "## Classifications" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "### Radio (single-choice)" ], "cell_type": "markdown" }, @@ -101,7 +106,7 @@ { "metadata": {}, "source": [ - "### Classification: checklist (multi-choice)" + "#### Checklist (multi-choice)" ], "cell_type": "markdown" }, diff --git a/examples/model_experiments/model_slices.ipynb b/examples/model_experiments/model_slices.ipynb index a91709fa0..91575a43e 100644 --- a/examples/model_experiments/model_slices.ipynb +++ b/examples/model_experiments/model_slices.ipynb @@ -42,7 +42,14 @@ }, { "metadata": {}, - "source": "%pip install labelbox", + "source": [ + "## Set up" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": "%pip install -q --upgrade \"labelbox[data]\"", "cell_type": "code", "outputs": [], "execution_count": null @@ -57,14 +64,14 @@ { "metadata": {}, "source": [ - "## API Key and Client\n", - "See the developer guide for [creating an API key](https://docs.labelbox.com/reference/create-api-key)." + "## API key and client\n", + "Provide a valid API key below to connect to the Labelbox client properly. For more information, please review the [Create API key](https://docs.labelbox.com/reference/create-api-key) guide." ], "cell_type": "markdown" }, { "metadata": {}, - "source": "# Add your API key\nAPI_KEY = \"\"\n# To get your API key go to: Workspace settings -> API -> Create API Key\nclient = lb.Client(api_key=API_KEY)", + "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", "cell_type": "code", "outputs": [], "execution_count": null @@ -110,7 +117,7 @@ }, { "metadata": {}, - "source": "model = client.create_model(name=\"Model Slice Demo\", ontology_id=ontology.uid)", + "source": "model = client.create_model(name=f\"Model Slice Demo {str(uuid.uuid4())}\",\n ontology_id=ontology.uid)", "cell_type": "code", "outputs": [], "execution_count": null @@ -240,6 +247,21 @@ "cell_type": "code", "outputs": [], "execution_count": null + }, + { + "metadata": {}, + "source": [ + "## Clean up\n", + "Uncomment and run the cell below to optionally delete Labelbox objects created." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": "# model_run.delete()\n# model.delete()\n# dataset.delete()", + "cell_type": "code", + "outputs": [], + "execution_count": null } ] } \ No newline at end of file diff --git a/examples/prediction_upload/pdf_predictions.ipynb b/examples/prediction_upload/pdf_predictions.ipynb index 942d40e9e..b50d0c3cc 100644 --- a/examples/prediction_upload/pdf_predictions.ipynb +++ b/examples/prediction_upload/pdf_predictions.ipynb @@ -256,7 +256,7 @@ }, { "metadata": {}, - "source": "client.enable_experimental = True\ntask = lb.DataRow.export(client=client, global_keys=[global_key])\ntask.wait_till_done()\nstream = task.get_buffered_stream()\n\ntext_layer = \"\"\nfor output in stream:\n output_json = json.loads(output.json)\n text_layer = output_json[\"media_attributes\"][\"text_layer_url\"]\nprint(text_layer)", + "source": "client.enable_experimental = True\ntask = lb.DataRow.export(client=client, global_keys=[global_key])\ntask.wait_till_done()\nstream = task.get_buffered_stream()\n\ntext_layer = \"\"\nfor output in stream:\n output_json = output.json\n text_layer = output_json[\"media_attributes\"][\"text_layer_url\"]\nprint(text_layer)", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/project_configuration/queue_management.ipynb b/examples/project_configuration/queue_management.ipynb index 0b62ea9d3..a4125386f 100644 --- a/examples/project_configuration/queue_management.ipynb +++ b/examples/project_configuration/queue_management.ipynb @@ -162,7 +162,7 @@ }, { "metadata": {}, - "source": "# Get data rows from project\ndata_rows = []\n\n\ndef json_stream_handler(output: lb.BufferedJsonConverterOutput):\n data_row = json.loads(output.json)\n data_rows.append(lb.GlobalKey(data_row[\"data_row\"][\"global_key\"])\n ) # Convert json data row into data row identifier object\n\n\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n export_json = export_task.get_buffered_stream(\n stream_type=lb.StreamType.RESULT).start(\n stream_handler=json_stream_handler)", + "source": "# Get data rows from project\ndata_rows = []\n\n\ndef json_stream_handler(output: lb.BufferedJsonConverterOutput):\n data_row = output.json\n data_rows.append(lb.GlobalKey(data_row[\"data_row\"][\"global_key\"])\n ) # Convert json data row into data row identifier object\n\n\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n export_json = export_task.get_buffered_stream(\n stream_type=lb.StreamType.RESULT).start(\n stream_handler=json_stream_handler)", "cell_type": "code", "outputs": [], "execution_count": null diff --git a/examples/template.ipynb b/examples/template.ipynb index f029217da..9a222f18c 100644 --- a/examples/template.ipynb +++ b/examples/template.ipynb @@ -1,18 +1,16 @@ { - "nbformat": 4, - "nbformat_minor": 2, - "metadata": {}, "cells": [ { + "cell_type": "markdown", "metadata": {}, "source": [ - "", - " ", + "\n", + " \n", "\n" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "\n", @@ -24,126 +22,152 @@ "\n", "" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "# Labelbox Example SDK Template\n", "\n", "This notebook serves as a template to guide develop of Labelbox example notebooks. Review notebook [CONTRIBUTING.md](https://github.com/Labelbox/labelbox-python/blob/develop/examples/CONTRIBUTING.md) for more detailed information." - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Set up" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "%pip install -q \"labelbox[data]\"\n# Always install \"labelbox[data]\" over labelbox to ensure dependencies are installed correctly\n# Include other installs in this block with similar format: \"%pip install -q \"", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "%pip install -q \"labelbox[data]\"\n", + "# Always install \"labelbox[data]\" over labelbox to ensure dependencies are installed correctly\n", + "# Include other installs in this block with similar format: \"%pip install -q \"" + ] }, { - "metadata": {}, - "source": "import labelbox as lb\n\n# Include other imports in this block try grouping similar imports together", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "import labelbox as lb\n", + "\n", + "# Include other imports in this block try grouping similar imports together" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## API key and client\n", "Provide a valid API key below in order to properly connect to the Labelbox client. Please review [Create API key guide](https://docs.labelbox.com/reference/create-api-key) for more information." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "API_KEY = None\n", + "client = lb.Client(api_key=API_KEY)" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Main section header\n", "\n", "* Include information on what section discusses" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Sub section header\n", "\n", "* Include information on what sub section discusses\n", "* If parameters are needed specify they need included" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "DATASET_ID = None\n# Inline Parameters should be all caps with a default value typically None", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "DATASET_ID = None\n", + "# Inline Parameters should be all caps with a default value typically None" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Notebook section with steps\n", "\n", "* For long winded processes step headers should be included" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Step \"step number\"" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "* Include information about step" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "#### \"step number.sub step number\"\n", "\n", "* Some steps need sub steps and should be labeled like above. For example, 7.1, 7.2, etc." - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Clean up\n", - "Uncomment and run the cell below to optionally delete Labelbox objects created" - ], - "cell_type": "markdown" + "Uncomment and run the cell below to optionally delete Labelbox objects created." + ] }, { - "metadata": {}, - "source": "# batch.delete()\n# project.delete()\n# dataset.delete()\n\n## Include other objects that were created", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "# batch.delete()\n", + "# project.delete()\n", + "# dataset.delete()\n", + "\n", + "## Include other objects that were created" + ] } - ] -} \ No newline at end of file + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}