From 3ccb9dc447535e863756c8ca6f35003bd713e02c Mon Sep 17 00:00:00 2001 From: Gabefire <33893811+Gabefire@users.noreply.github.com> Date: Fri, 31 May 2024 10:41:57 -0500 Subject: [PATCH 01/15] added new notebook : --- .../model_eveluation_output.ipynb | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 examples/project_configuration/model_eveluation_output.ipynb diff --git a/examples/project_configuration/model_eveluation_output.ipynb b/examples/project_configuration/model_eveluation_output.ipynb new file mode 100644 index 000000000..bef239999 --- /dev/null +++ b/examples/project_configuration/model_eveluation_output.ipynb @@ -0,0 +1,50 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Model Output Evaluation Project Setup\n", + "\n", + "This notebook will provide an example workflow of setting up a Model Output Evaluation (MOE) Project with the Labelbox-Python SDK.\n", + "MOE Projects are set up differently then other editors with it's own unique methods and modifications to existing methods:\n", + "\n", + "- `create_model_evaluation_project`: Main method used to create a chat model evaluation project\n", + "\n", + "- `create_ontology`: Methods used to create Labelbox ontologies for MOE project this requires a `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`\n", + "\n", + "- `create_ontology_from_feature_schemas`: Similar to `create_ontology` but from a list of `feature schema ids` which is designed to allow you to use existing features instead of creating new features. This also requires a `ontology_kind` set to `lb.OntologyKind.ModelEvaluation`." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "## Setup" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 9cf90d8393b920af7e1edd75316ec48e2b12d798 Mon Sep 17 00:00:00 2001 From: Gabefire <33893811+Gabefire@users.noreply.github.com> Date: Fri, 31 May 2024 13:01:10 -0500 Subject: [PATCH 02/15] finished notebook --- .../model_eveluation_output.ipynb | 50 ------------------- 1 file changed, 50 deletions(-) delete mode 100644 examples/project_configuration/model_eveluation_output.ipynb diff --git a/examples/project_configuration/model_eveluation_output.ipynb b/examples/project_configuration/model_eveluation_output.ipynb deleted file mode 100644 index bef239999..000000000 --- a/examples/project_configuration/model_eveluation_output.ipynb +++ /dev/null @@ -1,50 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# Model Output Evaluation Project Setup\n", - "\n", - "This notebook will provide an example workflow of setting up a Model Output Evaluation (MOE) Project with the Labelbox-Python SDK.\n", - "MOE Projects are set up differently then other editors with it's own unique methods and modifications to existing methods:\n", - "\n", - "- `create_model_evaluation_project`: Main method used to create a chat model evaluation project\n", - "\n", - "- `create_ontology`: Methods used to create Labelbox ontologies for MOE project this requires a `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`\n", - "\n", - "- `create_ontology_from_feature_schemas`: Similar to `create_ontology` but from a list of `feature schema ids` which is designed to allow you to use existing features instead of creating new features. This also requires a `ontology_kind` set to `lb.OntologyKind.ModelEvaluation`." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, - "outputs": [], - "source": [ - "## Setup" - ] - } - ], - "metadata": { - "language_info": { - "name": "python" - } - }, - "nbformat": 4, - "nbformat_minor": 2 -} From 5205a22609eacb8ecefcc7da265da5fc565c6cde Mon Sep 17 00:00:00 2001 From: Gabefire <33893811+Gabefire@users.noreply.github.com> Date: Fri, 31 May 2024 13:04:46 -0500 Subject: [PATCH 03/15] actually included notebook --- .../model_chat_evaluation_project.ipynb | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 examples/project_configuration/model_chat_evaluation_project.ipynb diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb new file mode 100644 index 000000000..b204bdac9 --- /dev/null +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -0,0 +1,197 @@ +{ + "nbformat": 4, + "nbformat_minor": 2, + "metadata": {}, + "cells": [ + { + "metadata": {}, + "source": [ + "", + " ", + "\n" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "# Model Chat Evaluation Project Setup\n", + "\n", + "This notebook will provide an example workflow of setting up a Model Chat Evaluation (MCE) Project with the Labelbox-Python SDK.\n", + "MOE Projects are set up differently then other editors with it's own unique method and modifications to existing methods:\n", + "\n", + "- `client.create_model_evaluation_project`: Main method used to create a model chat evaluation project\n", + "\n", + "- `client.create_ontology`: Methods used to create Labelbox ontologies for MCE project this requires a `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`\n", + "\n", + "- `client.create_ontology_from_feature_schemas`: Similar to `client.create_ontology` but from a list of `feature schema ids` which is designed to allow you to use existing features instead of creating new features. This also requires a `ontology_kind` set to `lb.OntologyKind.ModelEvaluation`." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "## Set Up" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": "%pip install -q \"labelbox[data]\"", + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": "import labelbox as lb", + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "## API Key and Client\n", + "Provide a valid API key below in order to properly connect to the Labelbox client. Please review [Create API key guide](https://docs.labelbox.com/reference/create-api-key) for more information." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "## Example: Create Model Chat Evaluation Project\n", + "\n", + "The steps to creating a Model Chat Evaluation Project through the Labelbox-Python SDK are similar to creating a regular project. However, they vary slightly, and we will showcase the different methods in this example workflow." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "### Create a MOE Ontology\n", + "\n", + "You can create ontologies for Model Evaluation projects the same way as creating ontologies for other projects with the only requirement of passing in a `ontology_kind` parameter which needs set to `lb.OntologyKind.ModelEvaluation`. You can create ontologies with two methods: `client.create_ontology` and `client.create_ontology_from_feature_schemas`." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "#### Option A: `client.create_ontology`\n", + "\n", + "Typically, you create ontologies and generate the associated features at the same time. Below is an example of creating an ontology for your model chat evaluation project using supported tools and classifications. For information on supported annotation types visit our [model chat evaluation](https://docs.labelbox.com/docs/model-chat-evaluation#supported-annotation-types) guide." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": "ontology_builder = lb.OntologyBuilder(\n tools=[\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n name=\"single select feature\",\n ),\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n name=\"multi select feature\",\n ),\n lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"ranking feature\"),\n ],\n classifications=[\n lb.Classification(\n class_type=lb.Classification.Type.CHECKLIST,\n name=\"checklist feature\",\n options=[\n lb.Option(value=\"option 1\", label=\"option 1\"),\n lb.Option(value=\"option 2\", label=\"option 2\"),\n ],\n ),\n lb.Classification(\n class_type=lb.Classification.Type.RADIO,\n name=\"radio_question\",\n options=[\n lb.Option(value=\"first_radio_answer\"),\n lb.Option(value=\"second_radio_answer\"),\n ],\n ),\n ],\n)\n\n# Create ontology\nontology = client.create_ontology(\n \"MCE ontology\",\n ontology_builder.asdict(),\n media_type=lb.MediaType.Conversational,\n ontology_kind=lb.OntologyKind.ModelEvaluation,\n)", + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "### Option B: `client.create_ontology_from_feature_schemas`\n", + "Ontologies can also be created with feature schema IDs. This makes your ontologies with existing features compared to generating new features. You can get these features by going to the _Schema_ tab inside Labelbox. (uncomment the below code block for this option)" + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": "# ontology = client.create_ontology_from_feature_schemas(\n# \"MCE ontology\",\n# feature_schema_ids=[\"\",\n media_type=lb.MediaType.Conversational,\n dataset_name=\"Demo MCE dataset\",\n data_row_count=100,\n)\n\n# Setup project with ontology created above\nproject.setup_editor(ontology)", + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "## Exporting Model Chat Evaluation Project\n", + "Exporting from a Model Chat Evaluation project works the same as exporting from other projects. In this example, unless you have created labels inside the Labelbox platform your exported will be shown as empty. Please review our [Model Chat Evaluation Export](https://docs.labelbox.com/reference/export-model-chat-evaluation-annotations) guide for a sample export." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": "# Start export from project\nexport_task = project.export()\nexport_task.wait_till_done()\n\n# Conditional if task has errors\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n # Start export stream\n stream = export_task.get_buffered_stream()\n\n # Iterate through data rows\n for data_row in stream:\n print(data_row.json)", + "cell_type": "code", + "outputs": [], + "execution_count": null + }, + { + "metadata": {}, + "source": [ + "## Clean Up\n", + "\n", + "This section serves as an optional clean-up step to delete the Labelbox assets created within this guide. You will need to uncomment the delete methods shown." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": "# project.delete()\n# client.delete_unused_ontology(ontology.uid)\n# dataset.delete()", + "cell_type": "code", + "outputs": [], + "execution_count": null + } + ] +} \ No newline at end of file From 157e27c5a057672e9d92e89fefe62f56d68afbc0 Mon Sep 17 00:00:00 2001 From: Gabefire <33893811+Gabefire@users.noreply.github.com> Date: Fri, 31 May 2024 13:05:17 -0500 Subject: [PATCH 04/15] made some modifications to template as well --- examples/template.ipynb | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/examples/template.ipynb b/examples/template.ipynb index 0411a416b..cb5761ea0 100644 --- a/examples/template.ipynb +++ b/examples/template.ipynb @@ -27,6 +27,22 @@ ], "cell_type": "markdown" }, + { + "metadata": {}, + "source": [ + "# Labelbox Example SDK Template\n", + "\n", + "This notebook serves as a template to guide develop of Labelbox example notebooks. Review notebook [CONTRIBUTING.md](https://github.com/Labelbox/labelbox-python/blob/develop/examples/CONTRIBUTING.md) for more detailed information." + ], + "cell_type": "markdown" + }, + { + "metadata": {}, + "source": [ + "## Set Up" + ], + "cell_type": "markdown" + }, { "metadata": {}, "source": "%pip install -q \"labelbox[data]\"\n# Always install \"labelbox[data]\" over labelbox to ensure dependencies are installed correctly\n# Include other installs in this block with similar format: \"%pip install -q \"", @@ -51,7 +67,7 @@ }, { "metadata": {}, - "source": "# Add your API key\nAPI_KEY = None\n# To get your API key go to: Workspace settings -> API -> Create API Key\nclient = lb.Client(api_key=API_KEY)", + "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", "cell_type": "code", "outputs": [], "execution_count": null From 44827fba6b39420acbfdae24b44e673af647d5b6 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 31 May 2024 18:07:20 +0000 Subject: [PATCH 05/15] :memo: README updated --- examples/README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/examples/README.md b/examples/README.md index 48580ecbb..478004fe3 100644 --- a/examples/README.md +++ b/examples/README.md @@ -104,6 +104,11 @@ + + Model Chat Evaluation Project + Open In Github + Open In Colab + Project Setup Open In Github From 7bd599d678ef32ece7452b06c1f906456fdc1b8c Mon Sep 17 00:00:00 2001 From: Gabefire <33893811+Gabefire@users.noreply.github.com> Date: Fri, 31 May 2024 13:29:59 -0500 Subject: [PATCH 06/15] remove description since it is not needed --- .../model_chat_evaluation_project.ipynb | 199 +++++++++++++----- 1 file changed, 144 insertions(+), 55 deletions(-) diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb index b204bdac9..d814c5b2d 100644 --- a/examples/project_configuration/model_chat_evaluation_project.ipynb +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -1,18 +1,16 @@ { - "nbformat": 4, - "nbformat_minor": 2, - "metadata": {}, "cells": [ { + "cell_type": "markdown", "metadata": {}, "source": [ - "", - " ", + "\n", + " \n", "\n" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "\n", @@ -24,10 +22,10 @@ "\n", "" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "# Model Chat Evaluation Project Setup\n", @@ -40,95 +38,149 @@ "- `client.create_ontology`: Methods used to create Labelbox ontologies for MCE project this requires a `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`\n", "\n", "- `client.create_ontology_from_feature_schemas`: Similar to `client.create_ontology` but from a list of `feature schema ids` which is designed to allow you to use existing features instead of creating new features. This also requires a `ontology_kind` set to `lb.OntologyKind.ModelEvaluation`." - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Set Up" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "%pip install -q \"labelbox[data]\"", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "%pip install -q \"labelbox[data]\"" + ] }, { - "metadata": {}, - "source": "import labelbox as lb", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "import labelbox as lb" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## API Key and Client\n", "Provide a valid API key below in order to properly connect to the Labelbox client. Please review [Create API key guide](https://docs.labelbox.com/reference/create-api-key) for more information." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "API_KEY = None\n", + "client = lb.Client(api_key=API_KEY)" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Example: Create Model Chat Evaluation Project\n", "\n", "The steps to creating a Model Chat Evaluation Project through the Labelbox-Python SDK are similar to creating a regular project. However, they vary slightly, and we will showcase the different methods in this example workflow." - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Create a MOE Ontology\n", "\n", "You can create ontologies for Model Evaluation projects the same way as creating ontologies for other projects with the only requirement of passing in a `ontology_kind` parameter which needs set to `lb.OntologyKind.ModelEvaluation`. You can create ontologies with two methods: `client.create_ontology` and `client.create_ontology_from_feature_schemas`." - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "#### Option A: `client.create_ontology`\n", "\n", "Typically, you create ontologies and generate the associated features at the same time. Below is an example of creating an ontology for your model chat evaluation project using supported tools and classifications. For information on supported annotation types visit our [model chat evaluation](https://docs.labelbox.com/docs/model-chat-evaluation#supported-annotation-types) guide." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "ontology_builder = lb.OntologyBuilder(\n tools=[\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n name=\"single select feature\",\n ),\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n name=\"multi select feature\",\n ),\n lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"ranking feature\"),\n ],\n classifications=[\n lb.Classification(\n class_type=lb.Classification.Type.CHECKLIST,\n name=\"checklist feature\",\n options=[\n lb.Option(value=\"option 1\", label=\"option 1\"),\n lb.Option(value=\"option 2\", label=\"option 2\"),\n ],\n ),\n lb.Classification(\n class_type=lb.Classification.Type.RADIO,\n name=\"radio_question\",\n options=[\n lb.Option(value=\"first_radio_answer\"),\n lb.Option(value=\"second_radio_answer\"),\n ],\n ),\n ],\n)\n\n# Create ontology\nontology = client.create_ontology(\n \"MCE ontology\",\n ontology_builder.asdict(),\n media_type=lb.MediaType.Conversational,\n ontology_kind=lb.OntologyKind.ModelEvaluation,\n)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "ontology_builder = lb.OntologyBuilder(\n", + " tools=[\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n", + " name=\"single select feature\",\n", + " ),\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n", + " name=\"multi select feature\",\n", + " ),\n", + " lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"ranking feature\"),\n", + " ],\n", + " classifications=[\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.CHECKLIST,\n", + " name=\"checklist feature\",\n", + " options=[\n", + " lb.Option(value=\"option 1\", label=\"option 1\"),\n", + " lb.Option(value=\"option 2\", label=\"option 2\"),\n", + " ],\n", + " ),\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.RADIO,\n", + " name=\"radio_question\",\n", + " options=[\n", + " lb.Option(value=\"first_radio_answer\"),\n", + " lb.Option(value=\"second_radio_answer\"),\n", + " ],\n", + " ),\n", + " ],\n", + ")\n", + "\n", + "# Create ontology\n", + "ontology = client.create_ontology(\n", + " \"MCE ontology\",\n", + " ontology_builder.asdict(),\n", + " media_type=lb.MediaType.Conversational,\n", + " ontology_kind=lb.OntologyKind.ModelEvaluation,\n", + ")" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Option B: `client.create_ontology_from_feature_schemas`\n", "Ontologies can also be created with feature schema IDs. This makes your ontologies with existing features compared to generating new features. You can get these features by going to the _Schema_ tab inside Labelbox. (uncomment the below code block for this option)" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "# ontology = client.create_ontology_from_feature_schemas(\n# \"MCE ontology\",\n# feature_schema_ids=[\"\",\n media_type=lb.MediaType.Conversational,\n dataset_name=\"Demo MCE dataset\",\n data_row_count=100,\n)\n\n# Setup project with ontology created above\nproject.setup_editor(ontology)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "project = client.create_model_evaluation_project(\n", + " name=\"Demo MCE Project\",\n", + " media_type=lb.MediaType.Conversational,\n", + " dataset_name=\"Demo MCE dataset\",\n", + " data_row_count=100,\n", + ")\n", + "\n", + "# Setup project with ontology created above\n", + "project.setup_editor(ontology)" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Exporting Model Chat Evaluation Project\n", "Exporting from a Model Chat Evaluation project works the same as exporting from other projects. In this example, unless you have created labels inside the Labelbox platform your exported will be shown as empty. Please review our [Model Chat Evaluation Export](https://docs.labelbox.com/reference/export-model-chat-evaluation-annotations) guide for a sample export." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "# Start export from project\nexport_task = project.export()\nexport_task.wait_till_done()\n\n# Conditional if task has errors\nif export_task.has_errors():\n export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n stream_handler=lambda error: print(error))\n\nif export_task.has_result():\n # Start export stream\n stream = export_task.get_buffered_stream()\n\n # Iterate through data rows\n for data_row in stream:\n print(data_row.json)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "# Start export from project\n", + "export_task = project.export()\n", + "export_task.wait_till_done()\n", + "\n", + "# Conditional if task has errors\n", + "if export_task.has_errors():\n", + " export_task.get_buffered_stream(stream_type=lb.StreamType.ERRORS).start(\n", + " stream_handler=lambda error: print(error))\n", + "\n", + "if export_task.has_result():\n", + " # Start export stream\n", + " stream = export_task.get_buffered_stream()\n", + "\n", + " # Iterate through data rows\n", + " for data_row in stream:\n", + " print(data_row.json)" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Clean Up\n", "\n", "This section serves as an optional clean-up step to delete the Labelbox assets created within this guide. You will need to uncomment the delete methods shown." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "# project.delete()\n# client.delete_unused_ontology(ontology.uid)\n# dataset.delete()", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "# project.delete()\n", + "# client.delete_unused_ontology(ontology.uid)\n", + "# dataset.delete()" + ] } - ] -} \ No newline at end of file + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From 497f4f7a156496d50d33bf2f2ef27d7346f4bcbb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 31 May 2024 18:31:31 +0000 Subject: [PATCH 07/15] :art: Cleaned --- .../model_chat_evaluation_project.ipynb | 199 +++++------------- 1 file changed, 55 insertions(+), 144 deletions(-) diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb index d814c5b2d..ef41d0fca 100644 --- a/examples/project_configuration/model_chat_evaluation_project.ipynb +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -1,16 +1,18 @@ { + "nbformat": 4, + "nbformat_minor": 2, + "metadata": {}, "cells": [ { - "cell_type": "markdown", "metadata": {}, "source": [ - "\n", - " \n", + "", + " ", "\n" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "\n", @@ -22,10 +24,10 @@ "\n", "" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "# Model Chat Evaluation Project Setup\n", @@ -38,149 +40,95 @@ "- `client.create_ontology`: Methods used to create Labelbox ontologies for MCE project this requires a `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`\n", "\n", "- `client.create_ontology_from_feature_schemas`: Similar to `client.create_ontology` but from a list of `feature schema ids` which is designed to allow you to use existing features instead of creating new features. This also requires a `ontology_kind` set to `lb.OntologyKind.ModelEvaluation`." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "## Set Up" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, + "source": "%pip install -q \"labelbox[data]\"", + "cell_type": "code", "outputs": [], - "source": [ - "%pip install -q \"labelbox[data]\"" - ] + "execution_count": null }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, + "source": "import labelbox as lb", + "cell_type": "code", "outputs": [], - "source": [ - "import labelbox as lb" - ] + "execution_count": null }, { - "cell_type": "markdown", "metadata": {}, "source": [ "## API Key and Client\n", "Provide a valid API key below in order to properly connect to the Labelbox client. Please review [Create API key guide](https://docs.labelbox.com/reference/create-api-key) for more information." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, + "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", + "cell_type": "code", "outputs": [], - "source": [ - "API_KEY = None\n", - "client = lb.Client(api_key=API_KEY)" - ] + "execution_count": null }, { - "cell_type": "markdown", "metadata": {}, "source": [ "## Example: Create Model Chat Evaluation Project\n", "\n", "The steps to creating a Model Chat Evaluation Project through the Labelbox-Python SDK are similar to creating a regular project. However, they vary slightly, and we will showcase the different methods in this example workflow." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "### Create a MOE Ontology\n", "\n", "You can create ontologies for Model Evaluation projects the same way as creating ontologies for other projects with the only requirement of passing in a `ontology_kind` parameter which needs set to `lb.OntologyKind.ModelEvaluation`. You can create ontologies with two methods: `client.create_ontology` and `client.create_ontology_from_feature_schemas`." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "#### Option A: `client.create_ontology`\n", "\n", "Typically, you create ontologies and generate the associated features at the same time. Below is an example of creating an ontology for your model chat evaluation project using supported tools and classifications. For information on supported annotation types visit our [model chat evaluation](https://docs.labelbox.com/docs/model-chat-evaluation#supported-annotation-types) guide." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, + "source": "ontology_builder = lb.OntologyBuilder(\n tools=[\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n name=\"single select feature\",\n ),\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n name=\"multi select feature\",\n ),\n lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"ranking feature\"),\n ],\n classifications=[\n lb.Classification(\n class_type=lb.Classification.Type.CHECKLIST,\n name=\"checklist feature\",\n options=[\n lb.Option(value=\"option 1\", label=\"option 1\"),\n lb.Option(value=\"option 2\", label=\"option 2\"),\n ],\n ),\n lb.Classification(\n class_type=lb.Classification.Type.RADIO,\n name=\"radio_question\",\n options=[\n lb.Option(value=\"first_radio_answer\"),\n lb.Option(value=\"second_radio_answer\"),\n ],\n ),\n ],\n)\n\n# Create ontology\nontology = client.create_ontology(\n \"MCE ontology\",\n ontology_builder.asdict(),\n media_type=lb.MediaType.Conversational,\n ontology_kind=lb.OntologyKind.ModelEvaluation,\n)", + "cell_type": "code", "outputs": [], - "source": [ - "ontology_builder = lb.OntologyBuilder(\n", - " tools=[\n", - " lb.Tool(\n", - " tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n", - " name=\"single select feature\",\n", - " ),\n", - " lb.Tool(\n", - " tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n", - " name=\"multi select feature\",\n", - " ),\n", - " lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"ranking feature\"),\n", - " ],\n", - " classifications=[\n", - " lb.Classification(\n", - " class_type=lb.Classification.Type.CHECKLIST,\n", - " name=\"checklist feature\",\n", - " options=[\n", - " lb.Option(value=\"option 1\", label=\"option 1\"),\n", - " lb.Option(value=\"option 2\", label=\"option 2\"),\n", - " ],\n", - " ),\n", - " lb.Classification(\n", - " class_type=lb.Classification.Type.RADIO,\n", - " name=\"radio_question\",\n", - " options=[\n", - " lb.Option(value=\"first_radio_answer\"),\n", - " lb.Option(value=\"second_radio_answer\"),\n", - " ],\n", - " ),\n", - " ],\n", - ")\n", - "\n", - "# Create ontology\n", - "ontology = client.create_ontology(\n", - " \"MCE ontology\",\n", - " ontology_builder.asdict(),\n", - " media_type=lb.MediaType.Conversational,\n", - " ontology_kind=lb.OntologyKind.ModelEvaluation,\n", - ")" - ] + "execution_count": null }, { - "cell_type": "markdown", "metadata": {}, "source": [ "### Option B: `client.create_ontology_from_feature_schemas`\n", "Ontologies can also be created with feature schema IDs. This makes your ontologies with existing features compared to generating new features. You can get these features by going to the _Schema_ tab inside Labelbox. (uncomment the below code block for this option)" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, + "source": "# ontology = client.create_ontology_from_feature_schemas(\n# \"MCE ontology\",\n# feature_schema_ids=[\" Date: Fri, 31 May 2024 13:38:08 -0500 Subject: [PATCH 08/15] typos --- .../model_chat_evaluation_project.ipynb | 207 +++++++++++++----- 1 file changed, 148 insertions(+), 59 deletions(-) diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb index ef41d0fca..2ba5b8bba 100644 --- a/examples/project_configuration/model_chat_evaluation_project.ipynb +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -1,18 +1,16 @@ { - "nbformat": 4, - "nbformat_minor": 2, - "metadata": {}, "cells": [ { + "cell_type": "markdown", "metadata": {}, "source": [ - "", - " ", + "\n", + " \n", "\n" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "\n", @@ -24,111 +22,165 @@ "\n", "" - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "# Model Chat Evaluation Project Setup\n", "\n", "This notebook will provide an example workflow of setting up a Model Chat Evaluation (MCE) Project with the Labelbox-Python SDK.\n", - "MOE Projects are set up differently then other editors with it's own unique method and modifications to existing methods:\n", + "Model Chat Evaluation Projects are set up differently then other editors with it's own unique method and modifications to existing methods:\n", "\n", "- `client.create_model_evaluation_project`: Main method used to create a model chat evaluation project\n", "\n", "- `client.create_ontology`: Methods used to create Labelbox ontologies for MCE project this requires a `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`\n", "\n", "- `client.create_ontology_from_feature_schemas`: Similar to `client.create_ontology` but from a list of `feature schema ids` which is designed to allow you to use existing features instead of creating new features. This also requires a `ontology_kind` set to `lb.OntologyKind.ModelEvaluation`." - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Set Up" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "%pip install -q \"labelbox[data]\"", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "%pip install -q \"labelbox[data]\"" + ] }, { - "metadata": {}, - "source": "import labelbox as lb", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "import labelbox as lb" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## API Key and Client\n", "Provide a valid API key below in order to properly connect to the Labelbox client. Please review [Create API key guide](https://docs.labelbox.com/reference/create-api-key) for more information." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "API_KEY = None\n", + "client = lb.Client(api_key=API_KEY)" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "## Example: Create Model Chat Evaluation Project\n", "\n", "The steps to creating a Model Chat Evaluation Project through the Labelbox-Python SDK are similar to creating a regular project. However, they vary slightly, and we will showcase the different methods in this example workflow." - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ - "### Create a MOE Ontology\n", + "### Create a Model Chat Evaluation Ontology\n", "\n", "You can create ontologies for Model Evaluation projects the same way as creating ontologies for other projects with the only requirement of passing in a `ontology_kind` parameter which needs set to `lb.OntologyKind.ModelEvaluation`. You can create ontologies with two methods: `client.create_ontology` and `client.create_ontology_from_feature_schemas`." - ], - "cell_type": "markdown" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "#### Option A: `client.create_ontology`\n", "\n", "Typically, you create ontologies and generate the associated features at the same time. Below is an example of creating an ontology for your model chat evaluation project using supported tools and classifications. For information on supported annotation types visit our [model chat evaluation](https://docs.labelbox.com/docs/model-chat-evaluation#supported-annotation-types) guide." - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "ontology_builder = lb.OntologyBuilder(\n tools=[\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n name=\"single select feature\",\n ),\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n name=\"multi select feature\",\n ),\n lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"ranking feature\"),\n ],\n classifications=[\n lb.Classification(\n class_type=lb.Classification.Type.CHECKLIST,\n name=\"checklist feature\",\n options=[\n lb.Option(value=\"option 1\", label=\"option 1\"),\n lb.Option(value=\"option 2\", label=\"option 2\"),\n ],\n ),\n lb.Classification(\n class_type=lb.Classification.Type.RADIO,\n name=\"radio_question\",\n options=[\n lb.Option(value=\"first_radio_answer\"),\n lb.Option(value=\"second_radio_answer\"),\n ],\n ),\n ],\n)\n\n# Create ontology\nontology = client.create_ontology(\n \"MCE ontology\",\n ontology_builder.asdict(),\n media_type=lb.MediaType.Conversational,\n ontology_kind=lb.OntologyKind.ModelEvaluation,\n)", "cell_type": "code", + "execution_count": null, + "metadata": {}, "outputs": [], - "execution_count": null + "source": [ + "ontology_builder = lb.OntologyBuilder(\n", + " tools=[\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n", + " name=\"single select feature\",\n", + " ),\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n", + " name=\"multi select feature\",\n", + " ),\n", + " lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"ranking feature\"),\n", + " ],\n", + " classifications=[\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.CHECKLIST,\n", + " name=\"checklist feature\",\n", + " options=[\n", + " lb.Option(value=\"option 1\", label=\"option 1\"),\n", + " lb.Option(value=\"option 2\", label=\"option 2\"),\n", + " ],\n", + " ),\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.RADIO,\n", + " name=\"radio_question\",\n", + " options=[\n", + " lb.Option(value=\"first_radio_answer\"),\n", + " lb.Option(value=\"second_radio_answer\"),\n", + " ],\n", + " ),\n", + " ],\n", + ")\n", + "\n", + "# Create ontology\n", + "ontology = client.create_ontology(\n", + " \"MCE ontology\",\n", + " ontology_builder.asdict(),\n", + " media_type=lb.MediaType.Conversational,\n", + " ontology_kind=lb.OntologyKind.ModelEvaluation,\n", + ")" + ] }, { + "cell_type": "markdown", "metadata": {}, "source": [ "### Option B: `client.create_ontology_from_feature_schemas`\n", "Ontologies can also be created with feature schema IDs. This makes your ontologies with existing features compared to generating new features. You can get these features by going to the _Schema_ tab inside Labelbox. (uncomment the below code block for this option)" - ], - "cell_type": "markdown" + ] }, { - "metadata": {}, - "source": "# ontology = client.create_ontology_from_feature_schemas(\n# \"MCE ontology\",\n# feature_schema_ids=[\" Date: Fri, 31 May 2024 18:39:06 +0000 Subject: [PATCH 09/15] :art: Cleaned --- .../model_chat_evaluation_project.ipynb | 199 +++++------------- 1 file changed, 55 insertions(+), 144 deletions(-) diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb index 2ba5b8bba..51d64afd1 100644 --- a/examples/project_configuration/model_chat_evaluation_project.ipynb +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -1,16 +1,18 @@ { + "nbformat": 4, + "nbformat_minor": 2, + "metadata": {}, "cells": [ { - "cell_type": "markdown", "metadata": {}, "source": [ - "\n", - " \n", + "", + " ", "\n" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "\n", @@ -22,10 +24,10 @@ "\n", "" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "# Model Chat Evaluation Project Setup\n", @@ -38,149 +40,95 @@ "- `client.create_ontology`: Methods used to create Labelbox ontologies for MCE project this requires a `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`\n", "\n", "- `client.create_ontology_from_feature_schemas`: Similar to `client.create_ontology` but from a list of `feature schema ids` which is designed to allow you to use existing features instead of creating new features. This also requires a `ontology_kind` set to `lb.OntologyKind.ModelEvaluation`." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "## Set Up" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, + "source": "%pip install -q \"labelbox[data]\"", + "cell_type": "code", "outputs": [], - "source": [ - "%pip install -q \"labelbox[data]\"" - ] + "execution_count": null }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, + "source": "import labelbox as lb", + "cell_type": "code", "outputs": [], - "source": [ - "import labelbox as lb" - ] + "execution_count": null }, { - "cell_type": "markdown", "metadata": {}, "source": [ "## API Key and Client\n", "Provide a valid API key below in order to properly connect to the Labelbox client. Please review [Create API key guide](https://docs.labelbox.com/reference/create-api-key) for more information." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, + "source": "API_KEY = None\nclient = lb.Client(api_key=API_KEY)", + "cell_type": "code", "outputs": [], - "source": [ - "API_KEY = None\n", - "client = lb.Client(api_key=API_KEY)" - ] + "execution_count": null }, { - "cell_type": "markdown", "metadata": {}, "source": [ "## Example: Create Model Chat Evaluation Project\n", "\n", "The steps to creating a Model Chat Evaluation Project through the Labelbox-Python SDK are similar to creating a regular project. However, they vary slightly, and we will showcase the different methods in this example workflow." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "### Create a Model Chat Evaluation Ontology\n", "\n", "You can create ontologies for Model Evaluation projects the same way as creating ontologies for other projects with the only requirement of passing in a `ontology_kind` parameter which needs set to `lb.OntologyKind.ModelEvaluation`. You can create ontologies with two methods: `client.create_ontology` and `client.create_ontology_from_feature_schemas`." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", "metadata": {}, "source": [ "#### Option A: `client.create_ontology`\n", "\n", "Typically, you create ontologies and generate the associated features at the same time. Below is an example of creating an ontology for your model chat evaluation project using supported tools and classifications. For information on supported annotation types visit our [model chat evaluation](https://docs.labelbox.com/docs/model-chat-evaluation#supported-annotation-types) guide." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, + "source": "ontology_builder = lb.OntologyBuilder(\n tools=[\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n name=\"single select feature\",\n ),\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n name=\"multi select feature\",\n ),\n lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"ranking feature\"),\n ],\n classifications=[\n lb.Classification(\n class_type=lb.Classification.Type.CHECKLIST,\n name=\"checklist feature\",\n options=[\n lb.Option(value=\"option 1\", label=\"option 1\"),\n lb.Option(value=\"option 2\", label=\"option 2\"),\n ],\n ),\n lb.Classification(\n class_type=lb.Classification.Type.RADIO,\n name=\"radio_question\",\n options=[\n lb.Option(value=\"first_radio_answer\"),\n lb.Option(value=\"second_radio_answer\"),\n ],\n ),\n ],\n)\n\n# Create ontology\nontology = client.create_ontology(\n \"MCE ontology\",\n ontology_builder.asdict(),\n media_type=lb.MediaType.Conversational,\n ontology_kind=lb.OntologyKind.ModelEvaluation,\n)", + "cell_type": "code", "outputs": [], - "source": [ - "ontology_builder = lb.OntologyBuilder(\n", - " tools=[\n", - " lb.Tool(\n", - " tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n", - " name=\"single select feature\",\n", - " ),\n", - " lb.Tool(\n", - " tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n", - " name=\"multi select feature\",\n", - " ),\n", - " lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"ranking feature\"),\n", - " ],\n", - " classifications=[\n", - " lb.Classification(\n", - " class_type=lb.Classification.Type.CHECKLIST,\n", - " name=\"checklist feature\",\n", - " options=[\n", - " lb.Option(value=\"option 1\", label=\"option 1\"),\n", - " lb.Option(value=\"option 2\", label=\"option 2\"),\n", - " ],\n", - " ),\n", - " lb.Classification(\n", - " class_type=lb.Classification.Type.RADIO,\n", - " name=\"radio_question\",\n", - " options=[\n", - " lb.Option(value=\"first_radio_answer\"),\n", - " lb.Option(value=\"second_radio_answer\"),\n", - " ],\n", - " ),\n", - " ],\n", - ")\n", - "\n", - "# Create ontology\n", - "ontology = client.create_ontology(\n", - " \"MCE ontology\",\n", - " ontology_builder.asdict(),\n", - " media_type=lb.MediaType.Conversational,\n", - " ontology_kind=lb.OntologyKind.ModelEvaluation,\n", - ")" - ] + "execution_count": null }, { - "cell_type": "markdown", "metadata": {}, "source": [ "### Option B: `client.create_ontology_from_feature_schemas`\n", "Ontologies can also be created with feature schema IDs. This makes your ontologies with existing features compared to generating new features. You can get these features by going to the _Schema_ tab inside Labelbox. (uncomment the below code block for this option)" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "code", - "execution_count": null, "metadata": {}, + "source": "# ontology = client.create_ontology_from_feature_schemas(\n# \"MCE ontology\",\n# feature_schema_ids=[\" Date: Fri, 31 May 2024 13:43:08 -0500 Subject: [PATCH 10/15] Typos --- .../model_chat_evaluation_project.ipynb | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb index 51d64afd1..6affd5b69 100644 --- a/examples/project_configuration/model_chat_evaluation_project.ipynb +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -33,13 +33,13 @@ "# Model Chat Evaluation Project Setup\n", "\n", "This notebook will provide an example workflow of setting up a Model Chat Evaluation (MCE) Project with the Labelbox-Python SDK.\n", - "Model Chat Evaluation Projects are set up differently then other editors with it's own unique method and modifications to existing methods:\n", + "Model Chat Evaluation Projects are set up differently than other projects with its own unique method and modifications to existing methods:\n", "\n", - "- `client.create_model_evaluation_project`: Main method used to create a model chat evaluation project\n", + "- `client.create_model_evaluation_project`: The main method used to create a model chat evaluation project\n", "\n", - "- `client.create_ontology`: Methods used to create Labelbox ontologies for MCE project this requires a `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`\n", + "- `client.create_ontology`: Methods used to create Labelbox ontologies for MCE project this requires an `ontology_kind` parameter set to `lb.OntologyKind.ModelEvaluation`\n", "\n", - "- `client.create_ontology_from_feature_schemas`: Similar to `client.create_ontology` but from a list of `feature schema ids` which is designed to allow you to use existing features instead of creating new features. This also requires a `ontology_kind` set to `lb.OntologyKind.ModelEvaluation`." + "- `client.create_ontology_from_feature_schemas`: Similar to `client.create_ontology` but from a list of `feature schema ids` designed to allow you to use existing features instead of creating new features. This also requires an `ontology_kind` set to `lb.OntologyKind.ModelEvaluation`." ], "cell_type": "markdown" }, @@ -194,4 +194,4 @@ "execution_count": null } ] -} \ No newline at end of file +} From 6fe0147867c8dd2915e7491627c084b5856b3fd2 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 31 May 2024 18:44:01 +0000 Subject: [PATCH 11/15] :art: Cleaned --- .../project_configuration/model_chat_evaluation_project.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb index 6affd5b69..c43188621 100644 --- a/examples/project_configuration/model_chat_evaluation_project.ipynb +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -194,4 +194,4 @@ "execution_count": null } ] -} +} \ No newline at end of file From 6c2c9d9cc0c6380193dab65ef810fb1cd7ed447a Mon Sep 17 00:00:00 2001 From: Gabe <33893811+Gabefire@users.noreply.github.com> Date: Fri, 31 May 2024 13:47:55 -0500 Subject: [PATCH 12/15] Typos --- .../model_chat_evaluation_project.ipynb | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb index c43188621..75769a7db 100644 --- a/examples/project_configuration/model_chat_evaluation_project.ipynb +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -132,7 +132,7 @@ "metadata": {}, "source": [ "### Set Up Model Chat Evaluation Project\n", - "You do not have to create data rows with a model evaluation project instead they are generated for you when you create the project. The method you use to create your project is `client.create_model_evaluation_project` which takes the same parameters as the traditional `client.create_project` but with a few specific additional parameters. \n", + "You do not have to create data rows with a model evaluation project; instead, they are generated for you when you create the project. The method you use to create your project is `client.create_model_evaluation_project`, which takes the same parameters as the traditional `client.create_project` but with a few specific additional parameters. \n", "\n", "__Setting up different models that you are evaluating can only be done inside the Labelbox platform__\n", "\n", @@ -143,13 +143,13 @@ "\n", " - `name`: The name of your new project.\n", "\n", - " - `description`: Optional description of your project.\n", + " - `description`: An optional description of your project.\n", "\n", " - `media_type`: The type of assets that this project will accept. This should be set to lb.MediaType.Conversational\n", "\n", - " - `dataset_name`: The name of the dataset were the data rows that are generated will be located. Include this parameter only needed if wanting to create a new dataset.\n", + " - `dataset_name`: The name of the dataset where the generated data rows will be located. Include this parameter only needed if wanting to create a new dataset.\n", "\n", - " - `dataset_id`: A dataset ID of an existing Labelbox dataset. Include this parameter if you are wanting to append to an existing MCE dataset.\n", + " - `dataset_id`: An optional dataset ID of an existing Labelbox dataset. Include this parameter if you are wanting to append to an existing MCE dataset.\n", "\n", " - `data_row_count`: The number of data row assets that will be generated and used with your project.\n" ], @@ -166,7 +166,7 @@ "metadata": {}, "source": [ "## Exporting Model Chat Evaluation Project\n", - "Exporting from a Model Chat Evaluation project works the same as exporting from other projects. In this example, unless you have created labels inside the Labelbox platform your export will be shown as empty. Please review our [Model Chat Evaluation Export](https://docs.labelbox.com/reference/export-model-chat-evaluation-annotations) guide for a sample export." + "Exporting from a Model Chat Evaluation project works the same as exporting from other projects. In this example, your export will be shown as empty unless you have created labels inside the Labelbox platform. Please review our [Model Chat Evaluation Export](https://docs.labelbox.com/reference/export-model-chat-evaluation-annotations) guide for a sample export." ], "cell_type": "markdown" }, @@ -194,4 +194,4 @@ "execution_count": null } ] -} \ No newline at end of file +} From f808befc90b0a77cda840e584b67fb2f384be5fa Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 31 May 2024 18:48:48 +0000 Subject: [PATCH 13/15] :art: Cleaned --- .../project_configuration/model_chat_evaluation_project.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb index 75769a7db..2e38d1de1 100644 --- a/examples/project_configuration/model_chat_evaluation_project.ipynb +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -194,4 +194,4 @@ "execution_count": null } ] -} +} \ No newline at end of file From cbe33fc392f0628a600f18858e8927c4151ebf9e Mon Sep 17 00:00:00 2001 From: Gabe <33893811+Gabefire@users.noreply.github.com> Date: Fri, 31 May 2024 13:52:00 -0500 Subject: [PATCH 14/15] Typo --- .../project_configuration/model_chat_evaluation_project.ipynb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb index 2e38d1de1..d215581ee 100644 --- a/examples/project_configuration/model_chat_evaluation_project.ipynb +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -147,7 +147,7 @@ "\n", " - `media_type`: The type of assets that this project will accept. This should be set to lb.MediaType.Conversational\n", "\n", - " - `dataset_name`: The name of the dataset where the generated data rows will be located. Include this parameter only needed if wanting to create a new dataset.\n", + " - `dataset_name`: The name of the dataset where the generated data rows will be located. Include this parameter only if you want to create a new dataset.\n", "\n", " - `dataset_id`: An optional dataset ID of an existing Labelbox dataset. Include this parameter if you are wanting to append to an existing MCE dataset.\n", "\n", @@ -194,4 +194,4 @@ "execution_count": null } ] -} \ No newline at end of file +} From a84bc6da91a40f91c3395753671421d4f9e312eb Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Fri, 31 May 2024 18:52:57 +0000 Subject: [PATCH 15/15] :art: Cleaned --- .../project_configuration/model_chat_evaluation_project.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/project_configuration/model_chat_evaluation_project.ipynb b/examples/project_configuration/model_chat_evaluation_project.ipynb index d215581ee..ede272b78 100644 --- a/examples/project_configuration/model_chat_evaluation_project.ipynb +++ b/examples/project_configuration/model_chat_evaluation_project.ipynb @@ -194,4 +194,4 @@ "execution_count": null } ] -} +} \ No newline at end of file