From fee8c9922f08a9e6f8a29b6a8ca878c84b88c56c Mon Sep 17 00:00:00 2001 From: x-eun Date: Thu, 12 Sep 2024 14:36:24 -0700 Subject: [PATCH 1/6] Create offline_multimodal_chat_evaluation.ipynb --- .../offline_multimodal_chat_evaluation.ipynb | 722 ++++++++++++++++++ 1 file changed, 722 insertions(+) create mode 100644 annotation_import/offline_multimodal_chat_evaluation.ipynb diff --git a/annotation_import/offline_multimodal_chat_evaluation.ipynb b/annotation_import/offline_multimodal_chat_evaluation.ipynb new file mode 100644 index 0000000..e4269ee --- /dev/null +++ b/annotation_import/offline_multimodal_chat_evaluation.ipynb @@ -0,0 +1,722 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "0eZsDkk8eNzq" + }, + "source": [ + " \n" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "3WhqOL9deNzs" + }, + "source": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5qUvNiiY_qhI" + }, + "source": [ + "# Offline multimodal chat evaluation annotation import\n", + "\n", + "This notebook provides examples of each annotation type supported by the offline multimodal chat evalution project and walks through the complete process of importing annotations as prelabels (model assisted labeling) or ground truth." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "tBcUvSRTeNzt" + }, + "source": [ + "# Set up" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "ty51UFk7eNzt", + "outputId": "0c1cbc21-d116-45ab-e53b-7287fdcc34ce" + }, + "outputs": [], + "source": [ + "%pip install -q \"labelbox[data]\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "yWt-qbgSeNzt" + }, + "outputs": [], + "source": [ + "import labelbox as lb\n", + "import labelbox.types as lb_types\n", + "import uuid" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "WDTOgrh6eNzu" + }, + "source": [ + "# Replace with your API key\n", + "\n", + "Replace the value of `API_KEY` with a valid [API key]([ref:create-api-key](https://docs.labelbox.com/reference/create-api-key) to connect to the Labelbox client." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "WJP9wgQ9eNzu" + }, + "outputs": [], + "source": [ + "API_KEY = \"\"\n", + "client = lb.Client(api_key=API_KEY)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "NehiK5inktza" + }, + "outputs": [], + "source": [ + "from labelbox.types import (\n", + " Label,\n", + " MessageEvaluationTaskAnnotation,\n", + " MessageInfo,\n", + " MessageMultiSelectionTask,\n", + " MessageRankingTask,\n", + " MessageSingleSelectionTask,\n", + " OrderedMessageInfo,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "_sffdTeveNzu" + }, + "source": [ + "# Supported annotations for multimodal chat evaluation" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vbyg4Ea0eNzu" + }, + "source": [ + "### Tool: Message ranking" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "EfbgAYZheNzu", + "outputId": "e9a93907-9b28-467e-d8a4-8cb4565d2c01" + }, + "outputs": [], + "source": [ + "message_ranking_annotation = lb_types.MessageEvaluationTaskAnnotation(\n", + " name=\"Message ranking\",\n", + " value=MessageRankingTask(\n", + " parent_message_id=\"clxfznjb800073b6v43ppx9ca\",\n", + " ranked_messages=[\n", + " OrderedMessageInfo(\n", + " message_id=\"clxfzocbm00083b6v8vczsept\",\n", + " model_config_name=\"GPT 4 with temperature 0.7\",\n", + " order=1,\n", + " ),\n", + " OrderedMessageInfo(\n", + " message_id=\"clxfzocbm00093b6vx4ndisub\",\n", + " model_config_name=\"GPT 5\",\n", + " order=2,\n", + " ),\n", + " ],\n", + " ),\n", + ")\n", + "\n", + "message_ranking_annotation_ndjson = {\n", + " \"name\": \"model output multi ranking\",\n", + " \"messageEvaluationTask\": {\n", + " \"format\": \"message-ranking\",\n", + " \"data\": {\n", + " \"parentMessageId\": \"clxfzhair00023b6vb607bqo6\",\n", + " \"rankedMessages\": [\n", + " {\n", + " \"messageId\": \"clxfzi3r400063b6vuaeajylo\",\n", + " \"modelConfigName\": \"GPT 4 with temperature 0.7\",\n", + " \"order\": 2,\n", + " },\n", + " {\n", + " \"messageId\": \"clxfzi3r400053b6vm5udpdgo\",\n", + " \"modelConfigName\": \"GPT 5\",\n", + " \"order\": 1,\n", + " },\n", + " ],\n", + " },\n", + " },\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "EdR7rFlMCQcl" + }, + "source": [ + "### Tool: Single message selection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "pMtx3Khi6KFb", + "outputId": "31ac7563-5474-491c-cda5-30da159fb57a" + }, + "outputs": [], + "source": [ + "single_message_selection_annotation = lb_types.MessageEvaluationTaskAnnotation(\n", + " name=\"Single message selection\",\n", + " value=MessageSingleSelectionTask(\n", + " message_id=\"clxfzi3r400053b6vm5udpdgo\",\n", + " parent_message_id=\"clxfzhair00023b6vb607bqo6\",\n", + " model_config_name=\"GPT 4 with temperature 0.7\",\n", + " ),\n", + ")\n", + "single_message_selection_annotation_ndjson = {\n", + " \"name\": \"Single message selection\",\n", + " \"messageEvaluationTask\": {\n", + " \"format\": \"message-single-selection\",\n", + " \"data\": {\n", + " \"messageId\": \"clxfzi3r400053b6vm5udpdgo\",\n", + " \"parentMessageId\": \"clxfzhair00023b6vb607bqo6\",\n", + " \"modelConfigName\": \"GPT 4 with temperature 0.7\",\n", + " },\n", + " },\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "miDD2em0CWsb" + }, + "source": [ + "### Tool: Multiple message selection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "xDFnlbVGjcPm" + }, + "outputs": [], + "source": [ + "multiple_message_selection_annotation = lb_types.MessageEvaluationTaskAnnotation(\n", + " name=\"Multi message selection\",\n", + " value=MessageMultiSelectionTask(\n", + " parent_message_id=\"clxfzhair00023b6vb607bqo6\",\n", + " selected_messages=[\n", + " MessageInfo(\n", + " message_id=\"clxfzi3r400063b6vuaeajylo\",\n", + " model_config_name=\"GPT 4 with temperature 0.7\",\n", + " ),\n", + " MessageInfo(\n", + " message_id=\"clxfzi3r400053b6vm5udpdgo\",\n", + " model_config_name=\"GPT 5\",\n", + " ),\n", + " ],\n", + " ),\n", + ")\n", + "multiple_message_selection_annotation_ndjson = {\n", + " \"name\": \"Multi message selection\",\n", + " \"messageEvaluationTask\": {\n", + " \"format\": \"message-multi-selection\",\n", + " \"data\": {\n", + " \"parentMessageId\": \"clxfzhair00023b6vb607bqo6\",\n", + " \"selectedMessages\": [\n", + " {\n", + " \"messageId\": \"clxfzi3r400063b6vuaeajylo\",\n", + " \"modelConfigName\": \"GPT 4 with temperature 0.7\",\n", + " },\n", + " {\n", + " \"messageId\": \"clxfzi3r400053b6vm5udpdgo\",\n", + " \"modelConfigName\": \"GPT 5\",\n", + " },\n", + " ],\n", + " },\n", + " },\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "wmRq_uUWeNzu" + }, + "source": [ + "### Classification: Radio (single-choice)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "lHNXIWSOeNzu" + }, + "outputs": [], + "source": [ + "radio_annotation = lb_types.ClassificationAnnotation(\n", + " name=\"Choose the best response\",\n", + " value=lb_types.Radio(answer=lb_types.ClassificationAnswer(\n", + " name=\"Response B\")),\n", + ")\n", + "\n", + "radio_annotation_ndjson = {\n", + " \"name\": \"Choose the best response\",\n", + " \"answer\": {\n", + " \"name\": \"Response B\"\n", + " },\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "vTUpRGWHeNzu" + }, + "source": [ + "### Classification: Free-form text" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "qNsiybbveNzu" + }, + "outputs": [], + "source": [ + "text_annotation = lb_types.ClassificationAnnotation(\n", + " name=\"Provide a reason for your choice\",\n", + " value=lb_types.Text(answer=\"the answer to the text questions right here\"),\n", + ")\n", + "\n", + "text_annotation_ndjson = {\n", + " \"name\": \"Provide a reason for your choice\",\n", + " \"answer\": \"This is the more concise answer\",\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "55D5csZpeNzv" + }, + "source": [ + "### Classification: Checklist (multi-choice)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YrpPFANVeNzv" + }, + "outputs": [], + "source": [ + "checklist_annotation = lb_types.ClassificationAnnotation(\n", + " name=\"checklist_convo\", # must match your ontology feature\"s name\n", + " value=lb_types.Checklist(answer=[\n", + " lb_types.ClassificationAnswer(name=\"first_checklist_answer\"),\n", + " lb_types.ClassificationAnswer(name=\"second_checklist_answer\"),\n", + " ]),\n", + " message_id=\"clxfznjb800073b6v43ppx9ca\", # Message specific annotation\n", + ")\n", + "\n", + "checklist_annotation_ndjson = {\n", + " \"name\": \"checklist_convo\",\n", + " \"answers\": [\n", + " {\n", + " \"name\": \"first_checklist_answer\"\n", + " },\n", + " {\n", + " \"name\": \"second_checklist_answer\"\n", + " },\n", + " ],\n", + " \"messageId\": \"clxfznjb800073b6v43ppx9ca\",\n", + "}" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "iLj37R6reNzv" + }, + "source": [ + "## Step 1: Import data rows into Catalog" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "x6KR44tAeNzv", + "outputId": "94e5185c-2ed3-4c6f-c5e2-3d253955775d" + }, + "outputs": [], + "source": [ + "mmc_asset = \"https://storage.googleapis.com/labelbox-datasets/conversational_model_evaluation_sample/offline-model-chat-evaluation.json\"\n", + "global_key = \"offline-multimodal_chat_evaluation\"\n", + "\n", + "# Upload data rows\n", + "convo_data = {\n", + " \"row_data\": mmc_asset,\n", + " \"global_key\": global_key\n", + "}\n", + "\n", + "# Create a dataset\n", + "dataset = client.create_dataset(name=\"offline-multimodal_chat_evaluation_demo\")\n", + "# Create a datarow\n", + "task = dataset.create_data_rows([convo_data])\n", + "task.wait_till_done()\n", + "print(\"Errors:\",task.errors)\n", + "print(\"Failed data rows:\", task.failed_data_rows)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "uw2288YNeNzv" + }, + "source": [ + "## Step 2: Create/select an Ontology" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "g2dqJApaeNzv" + }, + "outputs": [], + "source": [ + "ontology_builder = lb.OntologyBuilder(\n", + " tools=[\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n", + " name=\"Single message selection\",\n", + " ),\n", + " lb.Tool(\n", + " tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n", + " name=\"Multi message selection\",\n", + " ),\n", + " lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"Message ranking\"),\n", + " ],\n", + " classifications=[\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.RADIO,\n", + " scope=lb.Classification.Scope.GLOBAL,\n", + " name=\"Choose the best response\",\n", + " options=[lb.Option(value=\"Response A\"), lb.Option(value=\"Response B\"), lb.Option(value=\"Tie\")]\n", + " ),\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.TEXT,\n", + " name=\"Provide a reason for your choice\"\n", + " ),\n", + " lb.Classification(\n", + " class_type=lb.Classification.Type.CHECKLIST,\n", + " scope=lb.Classification.Scope.INDEX,\n", + " name=\"checklist_convo\",\n", + " options=[\n", + " lb.Option(value=\"first_checklist_answer\"),\n", + " lb.Option(value=\"second_checklist_answer\")\n", + " ]\n", + " )\n", + " ]\n", + ")\n", + "# Create ontology\n", + "ontology = client.create_ontology(\n", + " \"MMC ontology\",\n", + " ontology_builder.asdict(),\n", + " media_type=lb.MediaType.Conversational,\n", + " ontology_kind=lb.OntologyKind.ModelEvaluation,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "j9L1kSnheNzv" + }, + "source": [ + "## Step 3: Create a labeling project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_qQm7Bh5eNzv" + }, + "outputs": [], + "source": [ + "# Create Labelbox project\n", + "project = client.create_offline_model_evaluation_project(\n", + " name=\"Offline MMC Import Demo\",\n", + " description=\"\", # optional\n", + " media_type=lb.MediaType.Conversational,\n", + ")\n", + "\n", + "# Setup your ontology\n", + "project.connect_ontology(\n", + " ontology) # Connect your ontology and editor to your project" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "DFBbimOPeNzw" + }, + "source": [ + "## Step 4: Send a batch of data rows to the project" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "DBNTNIYGeNzw", + "outputId": "f6089d9e-4e64-4a59-c09e-3078ec80afa8" + }, + "outputs": [], + "source": [ + "# Create a batch to send to your project\n", + "batch = project.create_batch(\n", + " \"first-batch-convo-demo\", # Each batch in a project must have a unique name\n", + " global_keys=[\n", + " global_key\n", + " ], # Paginated collection of data row objects, list of data row ids or global keys\n", + " priority=5, # priority between 1(Highest) - 5(lowest)\n", + ")\n", + "\n", + "print(\"Batch: \", batch)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "gBWD5t5EeNzw" + }, + "source": [ + "## Step 5: Create the annotations payload" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "pbD74v7neNzw" + }, + "source": [ + "Python annotation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "op5i6cNQeNzw" + }, + "outputs": [], + "source": [ + "label = []\n", + "label.append(\n", + " lb_types.Label(\n", + " data={\"global_key\": global_key},\n", + " annotations=[\n", + " message_ranking_annotation,\n", + " single_message_selection_annotation,\n", + " multiple_message_selection_annotation,\n", + " text_annotation,\n", + " checklist_annotation,\n", + " radio_annotation,\n", + " ],\n", + " ))" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "i0SiIp1reNzw" + }, + "source": [ + "NDJSON annotation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 198 + }, + "collapsed": true, + "id": "-k55rNxQeNzw", + "outputId": "37ba73a9-0e9a-408e-dfbc-fe03492f4c65" + }, + "outputs": [], + "source": [ + "label_ndjson = []\n", + "for annotations in [\n", + " message_ranking_annotation_ndjson,\n", + " single_message_selection_annotation_ndjson,\n", + " multiple_message_selection_annotation_ndjson,\n", + " text_annotation_ndjson,\n", + " checklist_annotation_ndjson,\n", + " radio_annotation_ndjson,\n", + "]:\n", + " annotations.update({\"dataRow\": {\"globalKey\": global_key}})\n", + " label_ndjson.append(annotations)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "Nkg5VDU7eNzw" + }, + "source": [ + "## Step 6: import annotations to a project as pre-labels or ground truth labels" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "5MzijtA3eNzw" + }, + "source": [ + "### Import as prelabels (model assisted labeling)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "f0Hu2UsQeNzw", + "outputId": "e91ce10a-94ab-4aef-e3ce-2b1b30a644c7" + }, + "outputs": [], + "source": [ + "# @title\n", + "upload_job = lb.MALPredictionImport.create_from_objects(\n", + " client=client,\n", + " project_id=project.uid,\n", + " name=f\"mal_job-{str(uuid.uuid4())}\",\n", + " predictions=label,\n", + ")\n", + "\n", + "upload_job.wait_until_done()\n", + "print(\"Errors:\", upload_job.errors)\n", + "print(\"Status of uploads: \", upload_job.statuses)" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kL9aD-aleNzw" + }, + "source": [ + "### Import as ground truth labels" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/" + }, + "collapsed": true, + "id": "1k2F_DNKeNzz", + "outputId": "23517a11-9b5d-4ab5-ad52-0907bd6bc7b5" + }, + "outputs": [], + "source": [ + "upload_job = lb.LabelImport.create_from_objects(\n", + " client=client,\n", + " project_id=project.uid,\n", + " name=\"label_import_job\" + str(uuid.uuid4()),\n", + " labels=label,\n", + ")\n", + "\n", + "upload_job.wait_until_done()\n", + "print(\"Errors:\", upload_job.errors)\n", + "print(\"Status of uploads: \", upload_job.statuses)" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} From 9bdf501a25f1849ad26dace0202bf1277bf5f5d7 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 12 Sep 2024 21:38:15 +0000 Subject: [PATCH 2/6] :art: Cleaned --- .../offline_multimodal_chat_evaluation.ipynb | 721 ++++-------------- requirements-dev.lock | 24 +- 2 files changed, 162 insertions(+), 583 deletions(-) diff --git a/annotation_import/offline_multimodal_chat_evaluation.ipynb b/annotation_import/offline_multimodal_chat_evaluation.ipynb index e4269ee..c4b8f80 100644 --- a/annotation_import/offline_multimodal_chat_evaluation.ipynb +++ b/annotation_import/offline_multimodal_chat_evaluation.ipynb @@ -1,722 +1,301 @@ { + "nbformat": 4, + "nbformat_minor": 0, + "metadata": {}, "cells": [ { - "cell_type": "markdown", - "metadata": { - "id": "0eZsDkk8eNzq" - }, + "metadata": {}, "source": [ - " \n" - ] + "", + " ", + "\n" + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", - "metadata": { - "id": "3WhqOL9deNzs" - }, + "metadata": {}, "source": [ "\n", - "\n", "\n", "\n", "\n", - "\n", "" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", - "metadata": { - "id": "5qUvNiiY_qhI" - }, + "metadata": {}, "source": [ "# Offline multimodal chat evaluation annotation import\n", "\n", "This notebook provides examples of each annotation type supported by the offline multimodal chat evalution project and walks through the complete process of importing annotations as prelabels (model assisted labeling) or ground truth." - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", - "metadata": { - "id": "tBcUvSRTeNzt" - }, + "metadata": {}, "source": [ "# Set up" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "%pip install -q \"labelbox[data]\"", "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "collapsed": true, - "id": "ty51UFk7eNzt", - "outputId": "0c1cbc21-d116-45ab-e53b-7287fdcc34ce" - }, "outputs": [], - "source": [ - "%pip install -q \"labelbox[data]\"" - ] + "execution_count": null }, { + "metadata": {}, + "source": "import labelbox as lb\nimport labelbox.types as lb_types\nimport uuid", "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "yWt-qbgSeNzt" - }, "outputs": [], - "source": [ - "import labelbox as lb\n", - "import labelbox.types as lb_types\n", - "import uuid" - ] + "execution_count": null }, { - "cell_type": "markdown", - "metadata": { - "id": "WDTOgrh6eNzu" - }, + "metadata": {}, "source": [ "# Replace with your API key\n", "\n", "Replace the value of `API_KEY` with a valid [API key]([ref:create-api-key](https://docs.labelbox.com/reference/create-api-key) to connect to the Labelbox client." - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "API_KEY = \"\"\nclient = lb.Client(api_key=API_KEY)", "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "WJP9wgQ9eNzu" - }, "outputs": [], - "source": [ - "API_KEY = \"\"\n", - "client = lb.Client(api_key=API_KEY)" - ] + "execution_count": null }, { + "metadata": {}, + "source": "from labelbox.types import (\n Label,\n MessageEvaluationTaskAnnotation,\n MessageInfo,\n MessageMultiSelectionTask,\n MessageRankingTask,\n MessageSingleSelectionTask,\n OrderedMessageInfo,\n)", "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "NehiK5inktza" - }, "outputs": [], - "source": [ - "from labelbox.types import (\n", - " Label,\n", - " MessageEvaluationTaskAnnotation,\n", - " MessageInfo,\n", - " MessageMultiSelectionTask,\n", - " MessageRankingTask,\n", - " MessageSingleSelectionTask,\n", - " OrderedMessageInfo,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "_sffdTeveNzu" - }, + "execution_count": null + }, + { + "metadata": {}, "source": [ "# Supported annotations for multimodal chat evaluation" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", - "metadata": { - "id": "vbyg4Ea0eNzu" - }, + "metadata": {}, "source": [ "### Tool: Message ranking" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "message_ranking_annotation = lb_types.MessageEvaluationTaskAnnotation(\n name=\"Message ranking\",\n value=MessageRankingTask(\n parent_message_id=\"clxfznjb800073b6v43ppx9ca\",\n ranked_messages=[\n OrderedMessageInfo(\n message_id=\"clxfzocbm00083b6v8vczsept\",\n model_config_name=\"GPT 4 with temperature 0.7\",\n order=1,\n ),\n OrderedMessageInfo(\n message_id=\"clxfzocbm00093b6vx4ndisub\",\n model_config_name=\"GPT 5\",\n order=2,\n ),\n ],\n ),\n)\n\nmessage_ranking_annotation_ndjson = {\n \"name\": \"model output multi ranking\",\n \"messageEvaluationTask\": {\n \"format\": \"message-ranking\",\n \"data\": {\n \"parentMessageId\":\n \"clxfzhair00023b6vb607bqo6\",\n \"rankedMessages\": [\n {\n \"messageId\": \"clxfzi3r400063b6vuaeajylo\",\n \"modelConfigName\": \"GPT 4 with temperature 0.7\",\n \"order\": 2,\n },\n {\n \"messageId\": \"clxfzi3r400053b6vm5udpdgo\",\n \"modelConfigName\": \"GPT 5\",\n \"order\": 1,\n },\n ],\n },\n },\n}", "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "collapsed": true, - "id": "EfbgAYZheNzu", - "outputId": "e9a93907-9b28-467e-d8a4-8cb4565d2c01" - }, "outputs": [], - "source": [ - "message_ranking_annotation = lb_types.MessageEvaluationTaskAnnotation(\n", - " name=\"Message ranking\",\n", - " value=MessageRankingTask(\n", - " parent_message_id=\"clxfznjb800073b6v43ppx9ca\",\n", - " ranked_messages=[\n", - " OrderedMessageInfo(\n", - " message_id=\"clxfzocbm00083b6v8vczsept\",\n", - " model_config_name=\"GPT 4 with temperature 0.7\",\n", - " order=1,\n", - " ),\n", - " OrderedMessageInfo(\n", - " message_id=\"clxfzocbm00093b6vx4ndisub\",\n", - " model_config_name=\"GPT 5\",\n", - " order=2,\n", - " ),\n", - " ],\n", - " ),\n", - ")\n", - "\n", - "message_ranking_annotation_ndjson = {\n", - " \"name\": \"model output multi ranking\",\n", - " \"messageEvaluationTask\": {\n", - " \"format\": \"message-ranking\",\n", - " \"data\": {\n", - " \"parentMessageId\": \"clxfzhair00023b6vb607bqo6\",\n", - " \"rankedMessages\": [\n", - " {\n", - " \"messageId\": \"clxfzi3r400063b6vuaeajylo\",\n", - " \"modelConfigName\": \"GPT 4 with temperature 0.7\",\n", - " \"order\": 2,\n", - " },\n", - " {\n", - " \"messageId\": \"clxfzi3r400053b6vm5udpdgo\",\n", - " \"modelConfigName\": \"GPT 5\",\n", - " \"order\": 1,\n", - " },\n", - " ],\n", - " },\n", - " },\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "EdR7rFlMCQcl" - }, + "execution_count": null + }, + { + "metadata": {}, "source": [ "### Tool: Single message selection" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "single_message_selection_annotation = lb_types.MessageEvaluationTaskAnnotation(\n name=\"Single message selection\",\n value=MessageSingleSelectionTask(\n message_id=\"clxfzi3r400053b6vm5udpdgo\",\n parent_message_id=\"clxfzhair00023b6vb607bqo6\",\n model_config_name=\"GPT 4 with temperature 0.7\",\n ),\n)\nsingle_message_selection_annotation_ndjson = {\n \"name\": \"Single message selection\",\n \"messageEvaluationTask\": {\n \"format\": \"message-single-selection\",\n \"data\": {\n \"messageId\": \"clxfzi3r400053b6vm5udpdgo\",\n \"parentMessageId\": \"clxfzhair00023b6vb607bqo6\",\n \"modelConfigName\": \"GPT 4 with temperature 0.7\",\n },\n },\n}", "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "collapsed": true, - "id": "pMtx3Khi6KFb", - "outputId": "31ac7563-5474-491c-cda5-30da159fb57a" - }, "outputs": [], - "source": [ - "single_message_selection_annotation = lb_types.MessageEvaluationTaskAnnotation(\n", - " name=\"Single message selection\",\n", - " value=MessageSingleSelectionTask(\n", - " message_id=\"clxfzi3r400053b6vm5udpdgo\",\n", - " parent_message_id=\"clxfzhair00023b6vb607bqo6\",\n", - " model_config_name=\"GPT 4 with temperature 0.7\",\n", - " ),\n", - ")\n", - "single_message_selection_annotation_ndjson = {\n", - " \"name\": \"Single message selection\",\n", - " \"messageEvaluationTask\": {\n", - " \"format\": \"message-single-selection\",\n", - " \"data\": {\n", - " \"messageId\": \"clxfzi3r400053b6vm5udpdgo\",\n", - " \"parentMessageId\": \"clxfzhair00023b6vb607bqo6\",\n", - " \"modelConfigName\": \"GPT 4 with temperature 0.7\",\n", - " },\n", - " },\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "miDD2em0CWsb" - }, + "execution_count": null + }, + { + "metadata": {}, "source": [ "### Tool: Multiple message selection" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "multiple_message_selection_annotation = (\n lb_types.MessageEvaluationTaskAnnotation(\n name=\"Multi message selection\",\n value=MessageMultiSelectionTask(\n parent_message_id=\"clxfzhair00023b6vb607bqo6\",\n selected_messages=[\n MessageInfo(\n message_id=\"clxfzi3r400063b6vuaeajylo\",\n model_config_name=\"GPT 4 with temperature 0.7\",\n ),\n MessageInfo(\n message_id=\"clxfzi3r400053b6vm5udpdgo\",\n model_config_name=\"GPT 5\",\n ),\n ],\n ),\n ))\nmultiple_message_selection_annotation_ndjson = {\n \"name\": \"Multi message selection\",\n \"messageEvaluationTask\": {\n \"format\": \"message-multi-selection\",\n \"data\": {\n \"parentMessageId\":\n \"clxfzhair00023b6vb607bqo6\",\n \"selectedMessages\": [\n {\n \"messageId\": \"clxfzi3r400063b6vuaeajylo\",\n \"modelConfigName\": \"GPT 4 with temperature 0.7\",\n },\n {\n \"messageId\": \"clxfzi3r400053b6vm5udpdgo\",\n \"modelConfigName\": \"GPT 5\",\n },\n ],\n },\n },\n}", "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "xDFnlbVGjcPm" - }, "outputs": [], - "source": [ - "multiple_message_selection_annotation = lb_types.MessageEvaluationTaskAnnotation(\n", - " name=\"Multi message selection\",\n", - " value=MessageMultiSelectionTask(\n", - " parent_message_id=\"clxfzhair00023b6vb607bqo6\",\n", - " selected_messages=[\n", - " MessageInfo(\n", - " message_id=\"clxfzi3r400063b6vuaeajylo\",\n", - " model_config_name=\"GPT 4 with temperature 0.7\",\n", - " ),\n", - " MessageInfo(\n", - " message_id=\"clxfzi3r400053b6vm5udpdgo\",\n", - " model_config_name=\"GPT 5\",\n", - " ),\n", - " ],\n", - " ),\n", - ")\n", - "multiple_message_selection_annotation_ndjson = {\n", - " \"name\": \"Multi message selection\",\n", - " \"messageEvaluationTask\": {\n", - " \"format\": \"message-multi-selection\",\n", - " \"data\": {\n", - " \"parentMessageId\": \"clxfzhair00023b6vb607bqo6\",\n", - " \"selectedMessages\": [\n", - " {\n", - " \"messageId\": \"clxfzi3r400063b6vuaeajylo\",\n", - " \"modelConfigName\": \"GPT 4 with temperature 0.7\",\n", - " },\n", - " {\n", - " \"messageId\": \"clxfzi3r400053b6vm5udpdgo\",\n", - " \"modelConfigName\": \"GPT 5\",\n", - " },\n", - " ],\n", - " },\n", - " },\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "wmRq_uUWeNzu" - }, + "execution_count": null + }, + { + "metadata": {}, "source": [ "### Classification: Radio (single-choice)" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "radio_annotation = lb_types.ClassificationAnnotation(\n name=\"Choose the best response\",\n value=lb_types.Radio(answer=lb_types.ClassificationAnswer(\n name=\"Response B\")),\n)\n\nradio_annotation_ndjson = {\n \"name\": \"Choose the best response\",\n \"answer\": {\n \"name\": \"Response B\"\n },\n}", "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "lHNXIWSOeNzu" - }, "outputs": [], - "source": [ - "radio_annotation = lb_types.ClassificationAnnotation(\n", - " name=\"Choose the best response\",\n", - " value=lb_types.Radio(answer=lb_types.ClassificationAnswer(\n", - " name=\"Response B\")),\n", - ")\n", - "\n", - "radio_annotation_ndjson = {\n", - " \"name\": \"Choose the best response\",\n", - " \"answer\": {\n", - " \"name\": \"Response B\"\n", - " },\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "vTUpRGWHeNzu" - }, + "execution_count": null + }, + { + "metadata": {}, "source": [ "### Classification: Free-form text" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "text_annotation = lb_types.ClassificationAnnotation(\n name=\"Provide a reason for your choice\",\n value=lb_types.Text(answer=\"the answer to the text questions right here\"),\n)\n\ntext_annotation_ndjson = {\n \"name\": \"Provide a reason for your choice\",\n \"answer\": \"This is the more concise answer\",\n}", "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "qNsiybbveNzu" - }, "outputs": [], - "source": [ - "text_annotation = lb_types.ClassificationAnnotation(\n", - " name=\"Provide a reason for your choice\",\n", - " value=lb_types.Text(answer=\"the answer to the text questions right here\"),\n", - ")\n", - "\n", - "text_annotation_ndjson = {\n", - " \"name\": \"Provide a reason for your choice\",\n", - " \"answer\": \"This is the more concise answer\",\n", - "}" - ] + "execution_count": null }, { - "cell_type": "markdown", - "metadata": { - "id": "55D5csZpeNzv" - }, + "metadata": {}, "source": [ "### Classification: Checklist (multi-choice)" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "checklist_annotation = lb_types.ClassificationAnnotation(\n name=\"checklist_convo\", # must match your ontology feature\"s name\n value=lb_types.Checklist(answer=[\n lb_types.ClassificationAnswer(name=\"first_checklist_answer\"),\n lb_types.ClassificationAnswer(name=\"second_checklist_answer\"),\n ]),\n message_id=\"clxfznjb800073b6v43ppx9ca\", # Message specific annotation\n)\n\nchecklist_annotation_ndjson = {\n \"name\": \"checklist_convo\",\n \"answers\": [\n {\n \"name\": \"first_checklist_answer\"\n },\n {\n \"name\": \"second_checklist_answer\"\n },\n ],\n \"messageId\": \"clxfznjb800073b6v43ppx9ca\",\n}", "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "YrpPFANVeNzv" - }, "outputs": [], - "source": [ - "checklist_annotation = lb_types.ClassificationAnnotation(\n", - " name=\"checklist_convo\", # must match your ontology feature\"s name\n", - " value=lb_types.Checklist(answer=[\n", - " lb_types.ClassificationAnswer(name=\"first_checklist_answer\"),\n", - " lb_types.ClassificationAnswer(name=\"second_checklist_answer\"),\n", - " ]),\n", - " message_id=\"clxfznjb800073b6v43ppx9ca\", # Message specific annotation\n", - ")\n", - "\n", - "checklist_annotation_ndjson = {\n", - " \"name\": \"checklist_convo\",\n", - " \"answers\": [\n", - " {\n", - " \"name\": \"first_checklist_answer\"\n", - " },\n", - " {\n", - " \"name\": \"second_checklist_answer\"\n", - " },\n", - " ],\n", - " \"messageId\": \"clxfznjb800073b6v43ppx9ca\",\n", - "}" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "iLj37R6reNzv" - }, + "execution_count": null + }, + { + "metadata": {}, "source": [ "## Step 1: Import data rows into Catalog" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "mmc_asset = \"https://storage.googleapis.com/labelbox-datasets/conversational_model_evaluation_sample/offline-model-chat-evaluation.json\"\nglobal_key = \"offline-multimodal_chat_evaluation\"\n\n# Upload data rows\nconvo_data = {\"row_data\": mmc_asset, \"global_key\": global_key}\n\n# Create a dataset\ndataset = client.create_dataset(name=\"offline-multimodal_chat_evaluation_demo\")\n# Create a datarow\ntask = dataset.create_data_rows([convo_data])\ntask.wait_till_done()\nprint(\"Errors:\", task.errors)\nprint(\"Failed data rows:\", task.failed_data_rows)", "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "collapsed": true, - "id": "x6KR44tAeNzv", - "outputId": "94e5185c-2ed3-4c6f-c5e2-3d253955775d" - }, "outputs": [], - "source": [ - "mmc_asset = \"https://storage.googleapis.com/labelbox-datasets/conversational_model_evaluation_sample/offline-model-chat-evaluation.json\"\n", - "global_key = \"offline-multimodal_chat_evaluation\"\n", - "\n", - "# Upload data rows\n", - "convo_data = {\n", - " \"row_data\": mmc_asset,\n", - " \"global_key\": global_key\n", - "}\n", - "\n", - "# Create a dataset\n", - "dataset = client.create_dataset(name=\"offline-multimodal_chat_evaluation_demo\")\n", - "# Create a datarow\n", - "task = dataset.create_data_rows([convo_data])\n", - "task.wait_till_done()\n", - "print(\"Errors:\",task.errors)\n", - "print(\"Failed data rows:\", task.failed_data_rows)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "uw2288YNeNzv" - }, + "execution_count": null + }, + { + "metadata": {}, "source": [ "## Step 2: Create/select an Ontology" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "ontology_builder = lb.OntologyBuilder(\n tools=[\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n name=\"Single message selection\",\n ),\n lb.Tool(\n tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n name=\"Multi message selection\",\n ),\n lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"Message ranking\"),\n ],\n classifications=[\n lb.Classification(\n class_type=lb.Classification.Type.RADIO,\n scope=lb.Classification.Scope.GLOBAL,\n name=\"Choose the best response\",\n options=[\n lb.Option(value=\"Response A\"),\n lb.Option(value=\"Response B\"),\n lb.Option(value=\"Tie\"),\n ],\n ),\n lb.Classification(\n class_type=lb.Classification.Type.TEXT,\n name=\"Provide a reason for your choice\",\n ),\n lb.Classification(\n class_type=lb.Classification.Type.CHECKLIST,\n scope=lb.Classification.Scope.INDEX,\n name=\"checklist_convo\",\n options=[\n lb.Option(value=\"first_checklist_answer\"),\n lb.Option(value=\"second_checklist_answer\"),\n ],\n ),\n ],\n)\n# Create ontology\nontology = client.create_ontology(\n \"MMC ontology\",\n ontology_builder.asdict(),\n media_type=lb.MediaType.Conversational,\n ontology_kind=lb.OntologyKind.ModelEvaluation,\n)", "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "g2dqJApaeNzv" - }, "outputs": [], - "source": [ - "ontology_builder = lb.OntologyBuilder(\n", - " tools=[\n", - " lb.Tool(\n", - " tool=lb.Tool.Type.MESSAGE_SINGLE_SELECTION,\n", - " name=\"Single message selection\",\n", - " ),\n", - " lb.Tool(\n", - " tool=lb.Tool.Type.MESSAGE_MULTI_SELECTION,\n", - " name=\"Multi message selection\",\n", - " ),\n", - " lb.Tool(tool=lb.Tool.Type.MESSAGE_RANKING, name=\"Message ranking\"),\n", - " ],\n", - " classifications=[\n", - " lb.Classification(\n", - " class_type=lb.Classification.Type.RADIO,\n", - " scope=lb.Classification.Scope.GLOBAL,\n", - " name=\"Choose the best response\",\n", - " options=[lb.Option(value=\"Response A\"), lb.Option(value=\"Response B\"), lb.Option(value=\"Tie\")]\n", - " ),\n", - " lb.Classification(\n", - " class_type=lb.Classification.Type.TEXT,\n", - " name=\"Provide a reason for your choice\"\n", - " ),\n", - " lb.Classification(\n", - " class_type=lb.Classification.Type.CHECKLIST,\n", - " scope=lb.Classification.Scope.INDEX,\n", - " name=\"checklist_convo\",\n", - " options=[\n", - " lb.Option(value=\"first_checklist_answer\"),\n", - " lb.Option(value=\"second_checklist_answer\")\n", - " ]\n", - " )\n", - " ]\n", - ")\n", - "# Create ontology\n", - "ontology = client.create_ontology(\n", - " \"MMC ontology\",\n", - " ontology_builder.asdict(),\n", - " media_type=lb.MediaType.Conversational,\n", - " ontology_kind=lb.OntologyKind.ModelEvaluation,\n", - ")" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "j9L1kSnheNzv" - }, + "execution_count": null + }, + { + "metadata": {}, "source": [ "## Step 3: Create a labeling project" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "# Create Labelbox project\nproject = client.create_offline_model_evaluation_project(\n name=\"Offline MMC Import Demo\",\n description=\"\", # optional\n media_type=lb.MediaType.Conversational,\n)\n\n# Setup your ontology\nproject.connect_ontology(\n ontology) # Connect your ontology and editor to your project", "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "_qQm7Bh5eNzv" - }, "outputs": [], - "source": [ - "# Create Labelbox project\n", - "project = client.create_offline_model_evaluation_project(\n", - " name=\"Offline MMC Import Demo\",\n", - " description=\"\", # optional\n", - " media_type=lb.MediaType.Conversational,\n", - ")\n", - "\n", - "# Setup your ontology\n", - "project.connect_ontology(\n", - " ontology) # Connect your ontology and editor to your project" - ] + "execution_count": null }, { - "cell_type": "markdown", - "metadata": { - "id": "DFBbimOPeNzw" - }, + "metadata": {}, "source": [ "## Step 4: Send a batch of data rows to the project" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "# Create a batch to send to your project\nbatch = project.create_batch(\n \"first-batch-convo-demo\", # Each batch in a project must have a unique name\n global_keys=[\n global_key\n ], # Paginated collection of data row objects, list of data row ids or global keys\n priority=5, # priority between 1(Highest) - 5(lowest)\n)\n\nprint(\"Batch: \", batch)", "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "collapsed": true, - "id": "DBNTNIYGeNzw", - "outputId": "f6089d9e-4e64-4a59-c09e-3078ec80afa8" - }, "outputs": [], - "source": [ - "# Create a batch to send to your project\n", - "batch = project.create_batch(\n", - " \"first-batch-convo-demo\", # Each batch in a project must have a unique name\n", - " global_keys=[\n", - " global_key\n", - " ], # Paginated collection of data row objects, list of data row ids or global keys\n", - " priority=5, # priority between 1(Highest) - 5(lowest)\n", - ")\n", - "\n", - "print(\"Batch: \", batch)" - ] + "execution_count": null }, { - "cell_type": "markdown", - "metadata": { - "id": "gBWD5t5EeNzw" - }, + "metadata": {}, "source": [ "## Step 5: Create the annotations payload" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", - "metadata": { - "id": "pbD74v7neNzw" - }, + "metadata": {}, "source": [ "Python annotation" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "label = []\nlabel.append(\n lb_types.Label(\n data={\"global_key\": global_key},\n annotations=[\n message_ranking_annotation,\n single_message_selection_annotation,\n multiple_message_selection_annotation,\n text_annotation,\n checklist_annotation,\n radio_annotation,\n ],\n ))", "cell_type": "code", - "execution_count": null, - "metadata": { - "id": "op5i6cNQeNzw" - }, "outputs": [], - "source": [ - "label = []\n", - "label.append(\n", - " lb_types.Label(\n", - " data={\"global_key\": global_key},\n", - " annotations=[\n", - " message_ranking_annotation,\n", - " single_message_selection_annotation,\n", - " multiple_message_selection_annotation,\n", - " text_annotation,\n", - " checklist_annotation,\n", - " radio_annotation,\n", - " ],\n", - " ))" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "i0SiIp1reNzw" - }, + "execution_count": null + }, + { + "metadata": {}, "source": [ "NDJSON annotation" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "label_ndjson = []\nfor annotations in [\n message_ranking_annotation_ndjson,\n single_message_selection_annotation_ndjson,\n multiple_message_selection_annotation_ndjson,\n text_annotation_ndjson,\n checklist_annotation_ndjson,\n radio_annotation_ndjson,\n]:\n annotations.update({\"dataRow\": {\"globalKey\": global_key}})\n label_ndjson.append(annotations)", "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/", - "height": 198 - }, - "collapsed": true, - "id": "-k55rNxQeNzw", - "outputId": "37ba73a9-0e9a-408e-dfbc-fe03492f4c65" - }, "outputs": [], - "source": [ - "label_ndjson = []\n", - "for annotations in [\n", - " message_ranking_annotation_ndjson,\n", - " single_message_selection_annotation_ndjson,\n", - " multiple_message_selection_annotation_ndjson,\n", - " text_annotation_ndjson,\n", - " checklist_annotation_ndjson,\n", - " radio_annotation_ndjson,\n", - "]:\n", - " annotations.update({\"dataRow\": {\"globalKey\": global_key}})\n", - " label_ndjson.append(annotations)" - ] - }, - { - "cell_type": "markdown", - "metadata": { - "id": "Nkg5VDU7eNzw" - }, + "execution_count": null + }, + { + "metadata": {}, "source": [ "## Step 6: import annotations to a project as pre-labels or ground truth labels" - ] + ], + "cell_type": "markdown" }, { - "cell_type": "markdown", - "metadata": { - "id": "5MzijtA3eNzw" - }, + "metadata": {}, "source": [ "### Import as prelabels (model assisted labeling)" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "# @title\nupload_job = lb.MALPredictionImport.create_from_objects(\n client=client,\n project_id=project.uid,\n name=f\"mal_job-{str(uuid.uuid4())}\",\n predictions=label,\n)\n\nupload_job.wait_until_done()\nprint(\"Errors:\", upload_job.errors)\nprint(\"Status of uploads: \", upload_job.statuses)", "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "collapsed": true, - "id": "f0Hu2UsQeNzw", - "outputId": "e91ce10a-94ab-4aef-e3ce-2b1b30a644c7" - }, "outputs": [], - "source": [ - "# @title\n", - "upload_job = lb.MALPredictionImport.create_from_objects(\n", - " client=client,\n", - " project_id=project.uid,\n", - " name=f\"mal_job-{str(uuid.uuid4())}\",\n", - " predictions=label,\n", - ")\n", - "\n", - "upload_job.wait_until_done()\n", - "print(\"Errors:\", upload_job.errors)\n", - "print(\"Status of uploads: \", upload_job.statuses)" - ] + "execution_count": null }, { - "cell_type": "markdown", - "metadata": { - "id": "kL9aD-aleNzw" - }, + "metadata": {}, "source": [ "### Import as ground truth labels" - ] + ], + "cell_type": "markdown" }, { + "metadata": {}, + "source": "upload_job = lb.LabelImport.create_from_objects(\n client=client,\n project_id=project.uid,\n name=\"label_import_job\" + str(uuid.uuid4()),\n labels=label,\n)\n\nupload_job.wait_until_done()\nprint(\"Errors:\", upload_job.errors)\nprint(\"Status of uploads: \", upload_job.statuses)", "cell_type": "code", - "execution_count": null, - "metadata": { - "colab": { - "base_uri": "https://localhost:8080/" - }, - "collapsed": true, - "id": "1k2F_DNKeNzz", - "outputId": "23517a11-9b5d-4ab5-ad52-0907bd6bc7b5" - }, "outputs": [], - "source": [ - "upload_job = lb.LabelImport.create_from_objects(\n", - " client=client,\n", - " project_id=project.uid,\n", - " name=\"label_import_job\" + str(uuid.uuid4()),\n", - " labels=label,\n", - ")\n", - "\n", - "upload_job.wait_until_done()\n", - "print(\"Errors:\", upload_job.errors)\n", - "print(\"Status of uploads: \", upload_job.statuses)" - ] - } - ], - "metadata": { - "colab": { - "provenance": [] - }, - "kernelspec": { - "display_name": "Python 3", - "name": "python3" - }, - "language_info": { - "name": "python" + "execution_count": null } - }, - "nbformat": 4, - "nbformat_minor": 0 -} + ] +} \ No newline at end of file diff --git a/requirements-dev.lock b/requirements-dev.lock index ce6e8f2..d0f5cda 100644 --- a/requirements-dev.lock +++ b/requirements-dev.lock @@ -13,7 +13,7 @@ annotated-types==0.7.0 # via pydantic asttokens==2.4.1 # via stack-data -black==24.4.2 +black==24.8.0 click==8.1.7 # via black # via typer @@ -22,15 +22,15 @@ commonmark==0.9.1 databooks==1.3.10 decorator==5.1.1 # via ipython -executing==2.0.1 +executing==2.1.0 # via stack-data gitdb==4.0.11 # via gitpython gitpython==3.1.43 # via databooks -importlib-metadata==8.2.0 +importlib-metadata==8.5.0 # via yapf -ipython==8.26.0 +ipython==8.27.0 # via black jedi==0.19.1 # via ipython @@ -38,7 +38,7 @@ matplotlib-inline==0.1.7 # via ipython mypy-extensions==1.0.0 # via black -numpy==2.0.1 +numpy==2.1.1 # via pandas packaging==24.1 # via black @@ -49,7 +49,7 @@ pathspec==0.12.1 # via black pexpect==4.9.0 # via ipython -platformdirs==4.2.2 +platformdirs==4.3.2 # via black # via yapf prompt-toolkit==3.0.47 @@ -58,16 +58,16 @@ ptyprocess==0.7.0 # via pexpect pure-eval==0.2.3 # via stack-data -pydantic==2.8.2 +pydantic==2.9.1 # via databooks -pydantic-core==2.20.1 +pydantic-core==2.23.3 # via pydantic pygments==2.18.0 # via ipython # via rich python-dateutil==2.9.0.post0 # via pandas -pytz==2024.1 +pytz==2024.2 # via pandas rich==12.6.0 # via databooks @@ -81,7 +81,7 @@ smmap==5.0.1 # via gitdb stack-data==0.6.3 # via ipython -tokenize-rt==5.2.0 +tokenize-rt==6.0.0 # via black tomli==2.0.1 # via databooks @@ -89,7 +89,7 @@ tomli==2.0.1 traitlets==5.14.3 # via ipython # via matplotlib-inline -typer==0.12.3 +typer==0.12.5 # via databooks typing-extensions==4.12.2 # via databooks @@ -101,5 +101,5 @@ tzdata==2024.1 wcwidth==0.2.13 # via prompt-toolkit yapf==0.40.2 -zipp==3.19.2 +zipp==3.20.1 # via importlib-metadata From 86a47a9b785fa1e0c6a4f76dd0332f40230a70ba Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 12 Sep 2024 21:38:46 +0000 Subject: [PATCH 3/6] :memo: README updated --- README.md | 173 ++++++++++++++++++++++++++++-------------------------- 1 file changed, 89 insertions(+), 84 deletions(-) diff --git a/README.md b/README.md index 0b1f0f7..1affa88 100644 --- a/README.md +++ b/README.md @@ -77,6 +77,11 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe Open In Colab Open In Github + + Composite mask export + Open In Colab + Open In Github + Export data Open In Colab @@ -87,11 +92,6 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe Open In Colab Open In Github - - Composite mask export - Open In Colab - Open In Github - @@ -107,29 +107,29 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe - Project setup - Open In Colab - Open In Github + Prompt response projects + Open In Colab + Open In Github Queue management Open In Colab Open In Github - - Webhooks - Open In Colab - Open In Github - Multimodal chat project Open In Colab Open In Github - Prompt response projects - Open In Colab - Open In Github + Project setup + Open In Colab + Open In Github + + + Webhooks + Open In Colab + Open In Github @@ -146,9 +146,9 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe - Tiled - Open In Colab - Open In Github + PDF + Open In Colab + Open In Github Conversational LLM @@ -156,19 +156,19 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe Open In Github - HTML - Open In Colab - Open In Github + Text + Open In Colab + Open In Github - Image - Open In Colab - Open In Github + Video + Open In Colab + Open In Github - PDF - Open In Colab - Open In Github + Tiled + Open In Colab + Open In Github Prompt response @@ -176,24 +176,19 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe Open In Github - DICOM - Open In Colab - Open In Github - - - LLM data generation - Open In Colab - Open In Github + Audio + Open In Colab + Open In Github - Text - Open In Colab - Open In Github + Offline multimodal chat evaluation + Open In Colab + Open In Github - Audio - Open In Colab - Open In Github + Image + Open In Colab + Open In Github Conversational @@ -201,9 +196,19 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe Open In Github - Video - Open In Colab - Open In Github + LLM data generation + Open In Colab + Open In Github + + + HTML + Open In Colab + Open In Github + + + DICOM + Open In Colab + Open In Github @@ -220,29 +225,29 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe - Import YOLOv8 annotations - Open In Colab - Open In Github - - - Meta SAM video - Open In Colab - Open In Github + Huggingface custom embeddings + Open In Colab + Open In Github Meta SAM Open In Colab Open In Github + + Meta SAM video + Open In Colab + Open In Github + Langchain Open In Colab Open In Github - Huggingface custom embeddings - Open In Colab - Open In Github + Import YOLOv8 annotations + Open In Colab + Open In Github @@ -259,25 +264,25 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe - Custom metrics demo - Open In Colab - Open In Github + Custom metrics basics + Open In Colab + Open In Github Model slices Open In Colab Open In Github - - Custom metrics basics - Open In Colab - Open In Github - Model predictions to project Open In Colab Open In Github + + Custom metrics demo + Open In Colab + Open In Github + @@ -292,21 +297,6 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe - - Video predictions - Open In Colab - Open In Github - - - HTML predictions - Open In Colab - Open In Github - - - Geospatial predictions - Open In Colab - Open In Github - Conversational predictions Open In Colab @@ -317,21 +307,36 @@ Welcome to Labelbox Notebooks! These documents are directly linked from our Labe Open In Colab Open In Github - - Conversational LLM predictions - Open In Colab - Open In Github - PDF predictions Open In Colab Open In Github + + HTML predictions + Open In Colab + Open In Github + Image predictions Open In Colab Open In Github + + Geospatial predictions + Open In Colab + Open In Github + + + Video predictions + Open In Colab + Open In Github + + + Conversational LLM predictions + Open In Colab + Open In Github + From fb27453d5dd9dcba0922f8c32e994442e90a8acc Mon Sep 17 00:00:00 2001 From: x-eun Date: Thu, 12 Sep 2024 14:40:27 -0700 Subject: [PATCH 4/6] Update offline_multimodal_chat_evaluation.ipynb --- annotation_import/offline_multimodal_chat_evaluation.ipynb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/annotation_import/offline_multimodal_chat_evaluation.ipynb b/annotation_import/offline_multimodal_chat_evaluation.ipynb index e4269ee..d68c63c 100644 --- a/annotation_import/offline_multimodal_chat_evaluation.ipynb +++ b/annotation_import/offline_multimodal_chat_evaluation.ipynb @@ -16,7 +16,7 @@ }, "source": [ "\n", - "\n", "\n", "\n", From e9a7e69c35900148794043a6597550b9bbeb5e9d Mon Sep 17 00:00:00 2001 From: x-eun Date: Thu, 12 Sep 2024 14:46:59 -0700 Subject: [PATCH 5/6] Update offline_multimodal_chat_evaluation.ipynb --- annotation_import/offline_multimodal_chat_evaluation.ipynb | 1 - 1 file changed, 1 deletion(-) diff --git a/annotation_import/offline_multimodal_chat_evaluation.ipynb b/annotation_import/offline_multimodal_chat_evaluation.ipynb index d68c63c..5f30457 100644 --- a/annotation_import/offline_multimodal_chat_evaluation.ipynb +++ b/annotation_import/offline_multimodal_chat_evaluation.ipynb @@ -657,7 +657,6 @@ }, "outputs": [], "source": [ - "# @title\n", "upload_job = lb.MALPredictionImport.create_from_objects(\n", " client=client,\n", " project_id=project.uid,\n", From 6c61bd0e2b33c5bd128d6cd1f682b762b7f92052 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" Date: Thu, 12 Sep 2024 21:50:52 +0000 Subject: [PATCH 6/6] :art: Cleaned --- .../offline_multimodal_chat_evaluation.ipynb | 15 ++------------- 1 file changed, 2 insertions(+), 13 deletions(-) diff --git a/annotation_import/offline_multimodal_chat_evaluation.ipynb b/annotation_import/offline_multimodal_chat_evaluation.ipynb index 086c2c3..2eebec8 100644 --- a/annotation_import/offline_multimodal_chat_evaluation.ipynb +++ b/annotation_import/offline_multimodal_chat_evaluation.ipynb @@ -16,7 +16,7 @@ "metadata": {}, "source": [ "\n", - "\n", "\n", "\n", @@ -281,18 +281,7 @@ "source": "upload_job = lb.MALPredictionImport.create_from_objects(\n client=client,\n project_id=project.uid,\n name=f\"mal_job-{str(uuid.uuid4())}\",\n predictions=label,\n)\n\nupload_job.wait_until_done()\nprint(\"Errors:\", upload_job.errors)\nprint(\"Status of uploads: \", upload_job.statuses)", "cell_type": "code", "outputs": [], - "source": [ - "upload_job = lb.MALPredictionImport.create_from_objects(\n", - " client=client,\n", - " project_id=project.uid,\n", - " name=f\"mal_job-{str(uuid.uuid4())}\",\n", - " predictions=label,\n", - ")\n", - "\n", - "upload_job.wait_until_done()\n", - "print(\"Errors:\", upload_job.errors)\n", - "print(\"Status of uploads: \", upload_job.statuses)" - ] + "execution_count": null }, { "metadata": {},