diff --git a/argilla-server/src/argilla_server/_app.py b/argilla-server/src/argilla_server/_app.py
index 93edcd7580..295620c51c 100644
--- a/argilla-server/src/argilla_server/_app.py
+++ b/argilla-server/src/argilla_server/_app.py
@@ -298,7 +298,7 @@ def _show_telemetry_warning():
" https://docs.argilla.io/latest/reference/argilla-server/telemetry/\n\n"
"Telemetry is currently enabled. If you want to disable it, you can configure\n"
"the environment variable before relaunching the server:\n\n"
- f'{"#set HF_HUB_DISABLE_TELEMETRY=1" if os.name == "nt" else "$>export HF_HUB_DISABLE_TELEMETRY=1"}'
+ f"{'#set HF_HUB_DISABLE_TELEMETRY=1' if os.name == 'nt' else '$>export HF_HUB_DISABLE_TELEMETRY=1'}"
)
_LOGGER.warning(message)
diff --git a/argilla-v1/src/argilla_v1/client/datasets.py b/argilla-v1/src/argilla_v1/client/datasets.py
index 53c758a6b2..f12fbf69f4 100644
--- a/argilla-v1/src/argilla_v1/client/datasets.py
+++ b/argilla-v1/src/argilla_v1/client/datasets.py
@@ -1156,7 +1156,7 @@ def _prepare_for_training_with_spacy(self, nlp: "spacy.Language", records: List[
raise ValueError(
"The following annotation does not align with the tokens"
" produced by the provided spacy language model:"
- f" {(anno[0], record.text[anno[1]:anno[2]])}, {list(doc)}"
+ f" {(anno[0], record.text[anno[1] : anno[2]])}, {list(doc)}"
)
else:
entities.append(span)
diff --git a/argilla-v1/src/argilla_v1/client/feedback/dataset/helpers.py b/argilla-v1/src/argilla_v1/client/feedback/dataset/helpers.py
index 4580bb3ccc..782cf3783c 100644
--- a/argilla-v1/src/argilla_v1/client/feedback/dataset/helpers.py
+++ b/argilla-v1/src/argilla_v1/client/feedback/dataset/helpers.py
@@ -169,7 +169,7 @@ def normalize_records(
new_records.append(record)
else:
raise ValueError(
- "Expected `records` to be a list of `dict` or `FeedbackRecord`," f" got type `{type(record)}` instead."
+ f"Expected `records` to be a list of `dict` or `FeedbackRecord`, got type `{type(record)}` instead."
)
return new_records
@@ -384,7 +384,7 @@ def _validate_record_metadata(record: FeedbackRecord, metadata_schema: typing.Ty
metadata_schema.parse_obj(record.metadata)
except ValidationError as e:
raise ValueError(
- f"`FeedbackRecord.metadata` {record.metadata} does not match the expected schema," f" with exception: {e}"
+ f"`FeedbackRecord.metadata` {record.metadata} does not match the expected schema, with exception: {e}"
) from e
diff --git a/argilla-v1/src/argilla_v1/client/feedback/dataset/local/dataset.py b/argilla-v1/src/argilla_v1/client/feedback/dataset/local/dataset.py
index a3e4c687b1..f52e5627ec 100644
--- a/argilla-v1/src/argilla_v1/client/feedback/dataset/local/dataset.py
+++ b/argilla-v1/src/argilla_v1/client/feedback/dataset/local/dataset.py
@@ -230,7 +230,7 @@ def __getitem__(self, key: Union[slice, int]) -> Union["FeedbackRecord", List["F
"""
if len(self._records) < 1:
raise RuntimeError(
- "In order to get items from `FeedbackDataset` you need to add them first" " with `add_records`."
+ "In order to get items from `FeedbackDataset` you need to add them first with `add_records`."
)
if isinstance(key, int) and len(self._records) < key:
raise IndexError(f"This dataset contains {len(self)} records, so index {key} is out of range.")
@@ -331,8 +331,7 @@ def delete_vectors_settings(
if not self.vectors_settings:
raise ValueError(
- "The current `FeedbackDataset` does not contain any `vectors_settings` defined, so"
- " none can be deleted."
+ "The current `FeedbackDataset` does not contain any `vectors_settings` defined, so none can be deleted."
)
if not all(vector_setting in self._vectors_settings.keys() for vector_setting in vectors_settings):
diff --git a/argilla-v1/src/argilla_v1/client/feedback/dataset/local/mixins.py b/argilla-v1/src/argilla_v1/client/feedback/dataset/local/mixins.py
index 16be41313d..b351896dc6 100644
--- a/argilla-v1/src/argilla_v1/client/feedback/dataset/local/mixins.py
+++ b/argilla-v1/src/argilla_v1/client/feedback/dataset/local/mixins.py
@@ -89,7 +89,7 @@ def __delete_dataset(client: "httpx.Client", id: UUID) -> None:
datasets_api_v1.delete_dataset(client=client, id=id)
except Exception as e:
raise Exception(
- f"Failed while deleting the `FeedbackDataset` with ID '{id}' from Argilla with" f" exception: {e}"
+ f"Failed while deleting the `FeedbackDataset` with ID '{id}' from Argilla with exception: {e}"
) from e
@staticmethod
diff --git a/argilla-v1/src/argilla_v1/client/feedback/integrations/huggingface/model_card/model_card.py b/argilla-v1/src/argilla_v1/client/feedback/integrations/huggingface/model_card/model_card.py
index 48a26bbc78..db719d46ef 100644
--- a/argilla-v1/src/argilla_v1/client/feedback/integrations/huggingface/model_card/model_card.py
+++ b/argilla-v1/src/argilla_v1/client/feedback/integrations/huggingface/model_card/model_card.py
@@ -422,7 +422,7 @@ def generate(model_id: str, instruction: str, context: str = "") -> str:
)
return tokenizer.decode(outputs[0])
- generate("{self.output_dir.replace('"', '')}", "Is a toad a frog?")"""
+ generate("{self.output_dir.replace('"', "")}", "Is a toad a frog?")"""
)
elif self.task_type == "for_reward_modeling":
return predict_call + dedent(
diff --git a/argilla-v1/src/argilla_v1/client/models.py b/argilla-v1/src/argilla_v1/client/models.py
index 242434784d..7d4cc6e600 100644
--- a/argilla-v1/src/argilla_v1/client/models.py
+++ b/argilla-v1/src/argilla_v1/client/models.py
@@ -424,7 +424,7 @@ def __init__(
raise AssertionError("Missing fields: At least one of `text` or `tokens` argument must be provided!")
if (data.get("annotation") or data.get("prediction")) and text is None:
- raise AssertionError("Missing field `text`: " "char level spans must be provided with a raw text sentence")
+ raise AssertionError("Missing field `text`: char level spans must be provided with a raw text sentence")
if text is None:
text = " ".join(tokens)
diff --git a/argilla-v1/src/argilla_v1/client/sdk/commons/errors.py b/argilla-v1/src/argilla_v1/client/sdk/commons/errors.py
index f901fcce1d..359ddc79f3 100644
--- a/argilla-v1/src/argilla_v1/client/sdk/commons/errors.py
+++ b/argilla-v1/src/argilla_v1/client/sdk/commons/errors.py
@@ -26,7 +26,7 @@ def __init__(self, message: str, response: Any):
self.response = response
def __str__(self):
- return f"\nUnexpected response: {self.message}" "\nResponse content:" f"\n{self.response}"
+ return f"\nUnexpected response: {self.message}\nResponse content:\n{self.response}"
class InputValueError(BaseClientError):
@@ -52,7 +52,7 @@ def __init__(self, **ctx):
self.ctx = ctx
def __str__(self):
- return f"Argilla server returned an error with http status: {self.HTTP_STATUS}. " f"Error details: {self.ctx!r}"
+ return f"Argilla server returned an error with http status: {self.HTTP_STATUS}. Error details: {self.ctx!r}"
class BadRequestApiError(ArApiResponseError):
diff --git a/argilla-v1/src/argilla_v1/client/workspaces.py b/argilla-v1/src/argilla_v1/client/workspaces.py
index 119097452d..35d297528c 100644
--- a/argilla-v1/src/argilla_v1/client/workspaces.py
+++ b/argilla-v1/src/argilla_v1/client/workspaces.py
@@ -120,8 +120,7 @@ def users(self) -> List["UserModel"]:
def __repr__(self) -> str:
return (
- f"Workspace(id={self.id}, name={self.name},"
- f" inserted_at={self.inserted_at}, updated_at={self.updated_at})"
+ f"Workspace(id={self.id}, name={self.name}, inserted_at={self.inserted_at}, updated_at={self.updated_at})"
)
@allowed_for_roles(roles=[UserRole.owner])
@@ -330,8 +329,7 @@ def from_id(cls, id: UUID) -> "Workspace":
) from e
except ValidationApiError as e:
raise ValueError(
- "The ID you provided is not a valid UUID, so please make sure that the"
- " ID you provided is a valid one."
+ "The ID you provided is not a valid UUID, so please make sure that the ID you provided is a valid one."
) from e
except BaseClientError as e:
raise RuntimeError(f"Error while retrieving workspace with id=`{id}` from Argilla.") from e
diff --git a/argilla-v1/src/argilla_v1/labeling/text_classification/label_models.py b/argilla-v1/src/argilla_v1/labeling/text_classification/label_models.py
index 57ecb0cde8..a1ef315616 100644
--- a/argilla-v1/src/argilla_v1/labeling/text_classification/label_models.py
+++ b/argilla-v1/src/argilla_v1/labeling/text_classification/label_models.py
@@ -240,8 +240,7 @@ def _make_single_label_records(
pred_for_rec = [(self._weak_labels.labels[idx], prob[idx]) for idx in np.argsort(prob)[::-1]]
else:
raise NotImplementedError(
- f"The tie break policy '{tie_break_policy.value}' is not"
- f" implemented for {self.__class__.__name__}!"
+ f"The tie break policy '{tie_break_policy.value}' is not implemented for {self.__class__.__name__}!"
)
records_with_prediction.append(rec.copy(deep=True))
diff --git a/argilla-v1/src/argilla_v1/training/autotrain_advanced.py b/argilla-v1/src/argilla_v1/training/autotrain_advanced.py
index b291ead6c5..04edde074e 100644
--- a/argilla-v1/src/argilla_v1/training/autotrain_advanced.py
+++ b/argilla-v1/src/argilla_v1/training/autotrain_advanced.py
@@ -211,7 +211,7 @@ def __repr__(self):
formatted_string.append(arg_dict_key)
for idx, item in enumerate(arg_dict_single):
for key, val in item.items():
- formatted_string.append(f"\tjob{idx+1}-{key}: {val}")
+ formatted_string.append(f"\tjob{idx + 1}-{key}: {val}")
return "\n".join(formatted_string)
def train(self, output_dir: str):
diff --git a/argilla-v1/tests/integration/client/test_models.py b/argilla-v1/tests/integration/client/test_models.py
index 59451c0940..9d3a595732 100644
--- a/argilla-v1/tests/integration/client/test_models.py
+++ b/argilla-v1/tests/integration/client/test_models.py
@@ -130,7 +130,7 @@ def test_token_classification_with_tokens_and_tags(tokens, tags, annotation):
def test_token_classification_validations():
with pytest.raises(
AssertionError,
- match=("Missing fields: " "At least one of `text` or `tokens` argument must be provided!"),
+ match=("Missing fields: At least one of `text` or `tokens` argument must be provided!"),
):
TokenClassificationRecord()
@@ -138,13 +138,13 @@ def test_token_classification_validations():
annotation = [("test", 0, 4)]
with pytest.raises(
AssertionError,
- match=("Missing field `text`: " "char level spans must be provided with a raw text sentence"),
+ match=("Missing field `text`: char level spans must be provided with a raw text sentence"),
):
TokenClassificationRecord(tokens=tokens, annotation=annotation)
with pytest.raises(
AssertionError,
- match=("Missing field `text`: " "char level spans must be provided with a raw text sentence"),
+ match=("Missing field `text`: char level spans must be provided with a raw text sentence"),
):
TokenClassificationRecord(tokens=tokens, prediction=annotation)
diff --git a/argilla-v1/tests/unit/client/sdk/models/conftest.py b/argilla-v1/tests/unit/client/sdk/models/conftest.py
index 560b78c42c..b1ec76d6b1 100644
--- a/argilla-v1/tests/unit/client/sdk/models/conftest.py
+++ b/argilla-v1/tests/unit/client/sdk/models/conftest.py
@@ -45,7 +45,7 @@ def check_schema_props(client_props: dict, server_props: dict) -> bool:
continue
if name not in server_props:
LOGGER.warning(
- f"Client property {name} not found in server properties. " "Make sure your API compatibility"
+ f"Client property {name} not found in server properties. Make sure your API compatibility"
)
different_props.append(name)
continue
diff --git a/argilla/docs/community/integrations/llamaindex_rag_github.ipynb b/argilla/docs/community/integrations/llamaindex_rag_github.ipynb
index a19627f19a..8fd3c5c521 100644
--- a/argilla/docs/community/integrations/llamaindex_rag_github.ipynb
+++ b/argilla/docs/community/integrations/llamaindex_rag_github.ipynb
@@ -202,8 +202,7 @@
" \".svg\",\n",
" \".ico\",\n",
" \".json\",\n",
- " \".ipynb\", # Erase this line if you want to include notebooks\n",
- "\n",
+ " \".ipynb\", # Erase this line if you want to include notebooks\n",
" ],\n",
" GithubRepositoryReader.FilterType.EXCLUDE,\n",
" ),\n",
@@ -231,9 +230,7 @@
"outputs": [],
"source": [
"# LLM settings\n",
- "Settings.llm = OpenAI(\n",
- " model=\"gpt-3.5-turbo\", temperature=0.8, openai_api_key=openai_api_key\n",
- ")\n",
+ "Settings.llm = OpenAI(model=\"gpt-3.5-turbo\", temperature=0.8, openai_api_key=openai_api_key)\n",
"\n",
"# Load the data and create the index\n",
"index = VectorStoreIndex.from_documents(documents)\n",
diff --git a/argilla/docs/scripts/gen_popular_issues.py b/argilla/docs/scripts/gen_popular_issues.py
index d74bdeac1c..f6574fff11 100644
--- a/argilla/docs/scripts/gen_popular_issues.py
+++ b/argilla/docs/scripts/gen_popular_issues.py
@@ -116,21 +116,21 @@ def fetch_data_from_github(repository, auth_token):
f.write(" | Rank | Issue | Reactions | Comments |\n")
f.write(" |------|-------|:---------:|:--------:|\n")
for ix, row in engagement_df.iterrows():
- f.write(f" | {ix+1} | [{row['Issue']}]({row['URL']}) | 👍 {row['Reactions']} | 💬 {row['Comments']} |\n")
+ f.write(f" | {ix + 1} | [{row['Issue']}]({row['URL']}) | 👍 {row['Reactions']} | 💬 {row['Comments']} |\n")
f.write('\n=== "Latest issues open by the community"\n\n')
f.write(" | Rank | Issue | Author |\n")
f.write(" |------|-------|:------:|\n")
for ix, row in community_issues_df.iterrows():
state = "🟢" if row["State"] == "open" else "🟣"
- f.write(f" | {ix+1} | {state} [{row['Issue']}]({row['URL']}) | by **{row['Author']}** |\n")
+ f.write(f" | {ix + 1} | {state} [{row['Issue']}]({row['URL']}) | by **{row['Author']}** |\n")
f.write('\n=== "Planned issues for upcoming releases"\n\n')
f.write(" | Rank | Issue | Milestone |\n")
f.write(" |------|-------|:------:|\n")
for ix, row in planned_issues_df.iterrows():
state = "🟢" if row["State"] == "open" else "🟣"
- f.write(f" | {ix+1} | {state} [{row['Issue']}]({row['URL']}) | **{row['Milestone']}** |\n")
+ f.write(f" | {ix + 1} | {state} [{row['Issue']}]({row['URL']}) | **{row['Milestone']}** |\n")
today = datetime.today().date()
f.write(f"\nLast update: {today}\n")
diff --git a/argilla/docs/tutorials/image_classification.ipynb b/argilla/docs/tutorials/image_classification.ipynb
index d49b3fcf4c..46cdd909b0 100644
--- a/argilla/docs/tutorials/image_classification.ipynb
+++ b/argilla/docs/tutorials/image_classification.ipynb
@@ -93,13 +93,7 @@
"from PIL import Image\n",
"\n",
"from datasets import load_dataset, Dataset, load_metric\n",
- "from transformers import (\n",
- " AutoImageProcessor,\n",
- " AutoModelForImageClassification,\n",
- " pipeline,\n",
- " Trainer,\n",
- " TrainingArguments\n",
- ")\n",
+ "from transformers import AutoImageProcessor, AutoModelForImageClassification, pipeline, Trainer, TrainingArguments\n",
"\n",
"import argilla as rg"
]
@@ -182,7 +176,7 @@
" title=\"What digit do you see on the image?\",\n",
" labels=labels,\n",
" )\n",
- " ]\n",
+ " ],\n",
")"
]
},
@@ -246,7 +240,7 @@
"n_rows = 100\n",
"\n",
"hf_dataset = load_dataset(\"ylecun/mnist\", streaming=True)\n",
- "dataset_rows = [row for _,row in zip(range(n_rows), hf_dataset[\"train\"])]\n",
+ "dataset_rows = [row for _, row in zip(range(n_rows), hf_dataset[\"train\"])]\n",
"hf_dataset = Dataset.from_list(dataset_rows)\n",
"\n",
"hf_dataset"
@@ -525,7 +519,8 @@
],
"source": [
"def greyscale_to_rgb(img) -> Image:\n",
- " return Image.merge('RGB', (img, img, img))\n",
+ " return Image.merge(\"RGB\", (img, img, img))\n",
+ "\n",
"\n",
"submitted_image_rgb = [\n",
" {\n",
@@ -556,7 +551,7 @@
"\n",
"submitted_image_rgb_processed = [\n",
" {\n",
- " \"pixel_values\": processor(sample[\"image\"], return_tensors='pt')[\"pixel_values\"],\n",
+ " \"pixel_values\": processor(sample[\"image\"], return_tensors=\"pt\")[\"pixel_values\"],\n",
" \"label\": sample[\"label\"],\n",
" }\n",
" for sample in submitted_image_rgb\n",
@@ -624,8 +619,8 @@
"source": [
"def collate_fn(batch):\n",
" return {\n",
- " 'pixel_values': torch.stack([torch.tensor(x['pixel_values'][0]) for x in batch]),\n",
- " 'labels': torch.tensor([int(x['label']) for x in batch])\n",
+ " \"pixel_values\": torch.stack([torch.tensor(x[\"pixel_values\"][0]) for x in batch]),\n",
+ " \"labels\": torch.tensor([int(x[\"label\"]) for x in batch]),\n",
" }"
]
},
@@ -643,6 +638,8 @@
"outputs": [],
"source": [
"metric = load_metric(\"accuracy\", trust_remote_code=True)\n",
+ "\n",
+ "\n",
"def compute_metrics(p):\n",
" return metric.compute(predictions=np.argmax(p.predictions, axis=1), references=p.label_ids)"
]
@@ -664,7 +661,7 @@
" checkpoint,\n",
" num_labels=len(labels),\n",
" id2label={int(i): int(c) for i, c in enumerate(labels)},\n",
- " label2id={int(c): int(i) for i, c in enumerate(labels)}\n",
+ " label2id={int(c): int(i) for i, c in enumerate(labels)},\n",
")\n",
"model.config"
]
@@ -698,19 +695,19 @@
],
"source": [
"training_args = TrainingArguments(\n",
- " output_dir=\"./image-classifier\",\n",
- " per_device_train_batch_size=16,\n",
- " eval_strategy=\"steps\",\n",
- " num_train_epochs=1,\n",
- " fp16=False, # True if you have a GPU with mixed precision support\n",
- " save_steps=100,\n",
- " eval_steps=100,\n",
- " logging_steps=10,\n",
- " learning_rate=2e-4,\n",
- " save_total_limit=2,\n",
- " remove_unused_columns=True,\n",
- " push_to_hub=False,\n",
- " load_best_model_at_end=True,\n",
+ " output_dir=\"./image-classifier\",\n",
+ " per_device_train_batch_size=16,\n",
+ " eval_strategy=\"steps\",\n",
+ " num_train_epochs=1,\n",
+ " fp16=False, # True if you have a GPU with mixed precision support\n",
+ " save_steps=100,\n",
+ " eval_steps=100,\n",
+ " logging_steps=10,\n",
+ " learning_rate=2e-4,\n",
+ " save_total_limit=2,\n",
+ " remove_unused_columns=True,\n",
+ " push_to_hub=False,\n",
+ " load_best_model_at_end=True,\n",
")\n",
"\n",
"trainer = Trainer(\n",
@@ -745,12 +742,14 @@
"source": [
"pipe = pipeline(\"image-classification\", model=model, image_processor=processor)\n",
"\n",
+ "\n",
"def run_inference(batch):\n",
" predictions = pipe(batch[\"image\"])\n",
" batch[\"image_label\"] = [prediction[0][\"label\"] for prediction in predictions]\n",
" batch[\"score\"] = [prediction[0][\"score\"] for prediction in predictions]\n",
" return batch\n",
"\n",
+ "\n",
"hf_dataset = hf_dataset.map(run_inference, batched=True)"
]
},
diff --git a/argilla/docs/tutorials/image_preference.ipynb b/argilla/docs/tutorials/image_preference.ipynb
index 1f3aa9c43a..d25d433571 100644
--- a/argilla/docs/tutorials/image_preference.ipynb
+++ b/argilla/docs/tutorials/image_preference.ipynb
@@ -191,11 +191,10 @@
" metadata=[\n",
" rg.FloatMetadataProperty(name=\"toxicity\", title=\"Toxicity score\"),\n",
" rg.FloatMetadataProperty(name=\"identity_attack\", title=\"Identity attack score\"),\n",
- "\n",
" ],\n",
" vectors=[\n",
" rg.VectorField(name=\"original_caption_vector\", dimensions=384),\n",
- " ]\n",
+ " ],\n",
")"
]
},
@@ -254,7 +253,7 @@
"n_rows = 25\n",
"\n",
"hf_dataset = load_dataset(\"tomg-group-umd/pixelprose\", streaming=True)\n",
- "dataset_rows = [row for _,row in zip(range(n_rows), hf_dataset[\"train\"])]\n",
+ "dataset_rows = [row for _, row in zip(range(n_rows), hf_dataset[\"train\"])]\n",
"hf_dataset = Dataset.from_list(dataset_rows)\n",
"\n",
"hf_dataset"
@@ -341,8 +340,7 @@
}
],
"source": [
- "hf_dataset = hf_dataset.filter(\n",
- " lambda x: any([x[\"url\"].endswith(extension) for extension in [\".jpg\", \".png\", \".jpeg\"]]))\n",
+ "hf_dataset = hf_dataset.filter(lambda x: any([x[\"url\"].endswith(extension) for extension in [\".jpg\", \".png\", \".jpeg\"]]))\n",
"\n",
"hf_dataset"
]
@@ -380,6 +378,7 @@
"API_URL = \"https://api-inference.huggingface.co/models/black-forest-labs/FLUX.1-schnell\"\n",
"headers = {\"Authorization\": f\"Bearer {os.getenv('HF_TOKEN')}\"}\n",
"\n",
+ "\n",
"def query(payload):\n",
" response = requests.post(API_URL, headers=headers, json=payload)\n",
" if response.status_code == 200:\n",
@@ -391,9 +390,8 @@
" image = query(payload)\n",
" return image\n",
"\n",
- "query({\n",
- "\t\"inputs\": \"Astronaut riding a horse\"\n",
- "})"
+ "\n",
+ "query({\"inputs\": \"Astronaut riding a horse\"})"
]
},
{
@@ -426,9 +424,10 @@
"def generate_image(row):\n",
" caption = row[\"original_caption\"]\n",
" row[\"image_1\"] = query({\"inputs\": caption})\n",
- " row[\"image_2\"] = query({\"inputs\": caption + \" \"}) # space to avoid caching and getting the same image\n",
+ " row[\"image_2\"] = query({\"inputs\": caption + \" \"}) # space to avoid caching and getting the same image\n",
" return row\n",
- " \n",
+ "\n",
+ "\n",
"hf_dataset_with_images = hf_dataset.map(generate_image, batched=False)\n",
"\n",
"hf_dataset_with_images"
@@ -451,11 +450,13 @@
"source": [
"model = SentenceTransformer(\"TaylorAI/bge-micro-v2\")\n",
"\n",
+ "\n",
"def encode_questions(batch):\n",
" vectors_as_numpy = model.encode(batch[\"original_caption\"])\n",
" batch[\"original_caption_vector\"] = [x.tolist() for x in vectors_as_numpy]\n",
" return batch\n",
"\n",
+ "\n",
"hf_dataset_with_images_vectors = hf_dataset_with_images.map(encode_questions, batched=True)"
]
},
@@ -474,11 +475,14 @@
"metadata": {},
"outputs": [],
"source": [
- "dataset.records.log(records=hf_dataset_with_images_vectors, mapping={\n",
- " \"key\": \"id\",\n",
- " \"original_caption\": \"caption\",\n",
- " \"url\": \"image_original\",\n",
- "})"
+ "dataset.records.log(\n",
+ " records=hf_dataset_with_images_vectors,\n",
+ " mapping={\n",
+ " \"key\": \"id\",\n",
+ " \"original_caption\": \"caption\",\n",
+ " \"url\": \"image_original\",\n",
+ " },\n",
+ ")"
]
},
{
diff --git a/argilla/docs/tutorials/token_classification.ipynb b/argilla/docs/tutorials/token_classification.ipynb
index 71f457f69a..79d74c2dfe 100644
--- a/argilla/docs/tutorials/token_classification.ipynb
+++ b/argilla/docs/tutorials/token_classification.ipynb
@@ -309,9 +309,7 @@
"source": [
"def predict_gliner(model, text, labels, threshold):\n",
" entities = model.predict_entities(text, labels, threshold)\n",
- " return [\n",
- " {k: v for k, v in ent.items() if k not in {\"score\", \"text\"}} for ent in entities\n",
- " ]"
+ " return [{k: v for k, v in ent.items() if k not in {\"score\", \"text\"}} for ent in entities]"
]
},
{
@@ -330,9 +328,7 @@
"data = dataset.records.to_list(flatten=True)\n",
"updated_data = [\n",
" {\n",
- " \"span_label\": predict_gliner(\n",
- " model=gliner_model, text=sample[\"text\"], labels=labels, threshold=0.70\n",
- " ),\n",
+ " \"span_label\": predict_gliner(model=gliner_model, text=sample[\"text\"], labels=labels, threshold=0.70),\n",
" \"id\": sample[\"id\"],\n",
" }\n",
" for sample in data\n",
diff --git a/argilla/src/argilla/records/_io/_datasets.py b/argilla/src/argilla/records/_io/_datasets.py
index 50466c066a..dc6b35ca81 100644
--- a/argilla/src/argilla/records/_io/_datasets.py
+++ b/argilla/src/argilla/records/_io/_datasets.py
@@ -263,8 +263,7 @@ def to_argilla(hf_dataset: "HFDataset", mapper: "IngestedRecordMapper") -> "HFDa
if id_column_name not in hf_dataset.column_names:
split = hf_dataset.split
warnings.warn(
- message="Record id column not found in Hugging Face dataset. "
- "Using row index and split for record ids.",
+ message="Record id column not found in Hugging Face dataset. Using row index and split for record ids.",
)
hf_dataset = hf_dataset.map(
diff --git a/docs/_source/getting_started/quickstart_workflow.ipynb b/docs/_source/getting_started/quickstart_workflow.ipynb
index 9d13063ce3..1e84d37b6d 100644
--- a/docs/_source/getting_started/quickstart_workflow.ipynb
+++ b/docs/_source/getting_started/quickstart_workflow.ipynb
@@ -134,8 +134,8 @@
"outputs": [],
"source": [
"# Argilla credentials\n",
- "api_url = \"http://localhost:6900\" # \"https://.hf.space\"\n",
- "api_key = DEFAULT_API_KEY # admin.apikey\n",
+ "api_url = \"http://localhost:6900\" # \"https://.hf.space\"\n",
+ "api_key = DEFAULT_API_KEY # admin.apikey\n",
"# Huggingface credentials\n",
"hf_token = \"hf_...\""
]
@@ -177,6 +177,7 @@
},
{
"cell_type": "markdown",
+ "id": "7fb27b941602401d91542211134fc71a",
"metadata": {},
"source": [
"### Enable Telemetry\n",
@@ -187,14 +188,18 @@
{
"cell_type": "code",
"execution_count": null,
+ "id": "acae54e37e7d407bbb7b55eff062a284",
"metadata": {},
"outputs": [],
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -330,6 +335,7 @@
{
"cell_type": "code",
"execution_count": 36,
+ "id": "9a63283cbaf04dbcab1f6479b197f3a8",
"metadata": {},
"outputs": [
{
@@ -350,6 +356,7 @@
},
{
"cell_type": "markdown",
+ "id": "8dd0d8092fe74a7c96281538738b07e2",
"metadata": {},
"source": [
"As we can see, the dataset has two columns: `text` and `label`. We will use the label as the annotation of our record. Thus, to match the required attributes of a `TextClassificationRecord`, we need to rename the columns."
@@ -358,6 +365,7 @@
{
"cell_type": "code",
"execution_count": 76,
+ "id": "72eea5119410473aa328ad9291626812",
"metadata": {},
"outputs": [],
"source": [
@@ -366,6 +374,7 @@
},
{
"cell_type": "markdown",
+ "id": "8edb47106e1a46a883d545849b8ab81b",
"metadata": {},
"source": [
"Now, we can inspect our dataset."
@@ -374,6 +383,7 @@
{
"cell_type": "code",
"execution_count": 77,
+ "id": "10185d26023b46108eb7d9f57d49d2b3",
"metadata": {},
"outputs": [
{
@@ -439,6 +449,7 @@
},
{
"cell_type": "markdown",
+ "id": "8763a12b2bbd4a93a75aff182afb95dc",
"metadata": {},
"source": [
"Once, we checked that everything is correct, we can convert it to an Argilla dataset."
@@ -447,6 +458,7 @@
{
"cell_type": "code",
"execution_count": null,
+ "id": "7623eae2785240b9bd12b16a66d81610",
"metadata": {},
"outputs": [],
"source": [
@@ -621,6 +633,7 @@
},
{
"cell_type": "markdown",
+ "id": "7cdc8c89c7104fffa095e18ddfef8986",
"metadata": {},
"source": [
"As the label is not needed in this case, we will add it as metadata."
@@ -629,14 +642,16 @@
{
"cell_type": "code",
"execution_count": null,
+ "id": "b118ea5561624da68c537baed56e602f",
"metadata": {},
"outputs": [],
"source": [
"def metadata_to_dict(row):\n",
- " metadata = {}\n",
- " metadata[\"label\"] = row[\"label\"]\n",
- " row['metadata'] = metadata\n",
- " return row\n",
+ " metadata = {}\n",
+ " metadata[\"label\"] = row[\"label\"]\n",
+ " row[\"metadata\"] = metadata\n",
+ " return row\n",
+ "\n",
"\n",
"dataset = dataset.map(metadata_to_dict, remove_columns=[\"label\"])"
]
@@ -692,6 +707,7 @@
"# Load a english spaCy model to tokenize our text\n",
"nlp = spacy.load(\"en_core_web_sm\")\n",
"\n",
+ "\n",
"# Define our tokenize function\n",
"def tokenize(row):\n",
" tokens = [token.text for token in nlp(row[\"text\"])]\n",
@@ -915,7 +931,11 @@
"from datasets import load_dataset\n",
"\n",
"# Load the Dataset from the Hugging Face Hub and extract a subset of the train split as example\n",
- "dataset = load_dataset(\"europa_ecdc_tm\", \"en2fr\", split=\"train\").shuffle(seed=30).select(range(100))"
+ "dataset = (\n",
+ " load_dataset(\"europa_ecdc_tm\", \"en2fr\", split=\"train\")\n",
+ " .shuffle(seed=30)\n",
+ " .select(range(100))\n",
+ ")"
]
},
{
@@ -999,11 +1019,11 @@
"source": [
"# Define our helper extract function\n",
"def extract(row):\n",
- " return {\"text\": row[\"translation\"][\"en\"], \"prediction\":[row[\"translation\"][\"fr\"]]}\n",
+ " return {\"text\": row[\"translation\"][\"en\"], \"prediction\": [row[\"translation\"][\"fr\"]]}\n",
"\n",
"\n",
"# Map the extract function to our dataset\n",
- "dataset = dataset.map(extract, remove_columns = [\"translation\"])"
+ "dataset = dataset.map(extract, remove_columns=[\"translation\"])"
]
},
{
@@ -1463,10 +1483,7 @@
"sentence = \"I love this film, but the new remake is terrible.\"\n",
"\n",
"trainer = ArgillaTrainer(\n",
- " name=\"imdb\",\n",
- " workspace=\"argilla\",\n",
- " framework=\"spacy\",\n",
- " train_size=0.8\n",
+ " name=\"imdb\", workspace=\"argilla\", framework=\"spacy\", train_size=0.8\n",
")\n",
"trainer.update_config(max_epochs=1, max_steps=1)\n",
"trainer.train(output_dir=\"my_easy_model\")\n",
diff --git a/docs/_source/getting_started/quickstart_workflow_feedback.ipynb b/docs/_source/getting_started/quickstart_workflow_feedback.ipynb
index 1fdfcb624a..c26a23b16a 100644
--- a/docs/_source/getting_started/quickstart_workflow_feedback.ipynb
+++ b/docs/_source/getting_started/quickstart_workflow_feedback.ipynb
@@ -142,6 +142,7 @@
],
"source": [
"import argilla as rg\n",
+ "\n",
"rg.init(api_url=api_url, api_key=api_key)\n",
"\n",
"# # If you want to use your private HF Space\n",
@@ -150,6 +151,7 @@
},
{
"cell_type": "markdown",
+ "id": "7fb27b941602401d91542211134fc71a",
"metadata": {},
"source": [
"### Enable Telemetry\n",
@@ -160,14 +162,18 @@
{
"cell_type": "code",
"execution_count": null,
+ "id": "acae54e37e7d407bbb7b55eff062a284",
"metadata": {},
"outputs": [],
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -292,7 +298,7 @@
" fields={\n",
" \"text\": \"I feel sad today\",\n",
" },\n",
- " )\n",
+ " ),\n",
"]\n",
"dataset.add_records(records)"
]
@@ -349,7 +355,9 @@
"outputs": [],
"source": [
"# Besides Argilla, it can also be imported with load_dataset from datasets\n",
- "dataset_hf = rg.FeedbackDataset.from_huggingface(\"argilla/emotion\", split=\"train[1:101]\")"
+ "dataset_hf = rg.FeedbackDataset.from_huggingface(\n",
+ " \"argilla/emotion\", split=\"train[1:101]\"\n",
+ ")"
]
},
{
@@ -396,8 +404,7 @@
"from argilla.feedback import TrainingTask\n",
"\n",
"task = TrainingTask.for_text_classification(\n",
- " text=dataset_hf.field_by_name(\"text\"),\n",
- " label=dataset_hf.question_by_name(\"label\")\n",
+ " text=dataset_hf.field_by_name(\"text\"), label=dataset_hf.question_by_name(\"label\")\n",
")"
]
},
@@ -421,10 +428,7 @@
"from argilla.feedback import ArgillaTrainer\n",
"\n",
"trainer = ArgillaTrainer(\n",
- " dataset=dataset_hf,\n",
- " task=task,\n",
- " framework=\"setfit\",\n",
- " train_size=0.8\n",
+ " dataset=dataset_hf, task=task, framework=\"setfit\", train_size=0.8\n",
")"
]
},
diff --git a/docs/_source/practical_guides/annotation_workflows/add_text_descriptives_as_metadata.ipynb b/docs/_source/practical_guides/annotation_workflows/add_text_descriptives_as_metadata.ipynb
index 3eb9e80312..20ba8c2711 100644
--- a/docs/_source/practical_guides/annotation_workflows/add_text_descriptives_as_metadata.ipynb
+++ b/docs/_source/practical_guides/annotation_workflows/add_text_descriptives_as_metadata.ipynb
@@ -98,7 +98,9 @@
"outputs": [],
"source": [
"import argilla as rg\n",
- "from argilla.client.feedback.integrations.textdescriptives import TextDescriptivesExtractor\n",
+ "from argilla.client.feedback.integrations.textdescriptives import (\n",
+ " TextDescriptivesExtractor,\n",
+ ")\n",
"\n",
"from datasets import load_dataset"
]
@@ -119,11 +121,7 @@
"# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
"# Replace api_key if you configured a custom API key\n",
"# Replace workspace with the name of your workspace\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"owner.apikey\",\n",
- " workspace=\"admin\"\n",
- ")"
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"owner.apikey\", workspace=\"admin\")"
]
},
{
@@ -146,7 +144,7 @@
"# # Replace api_url with the url to your HF Spaces URL\n",
"# # Replace api_key if you configured a custom API key\n",
"# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
"# api_key=\"admin.apikey\",\n",
"# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
"# )"
@@ -169,9 +167,12 @@
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -333,10 +334,10 @@
"source": [
"# Initialize the TextDescriptivesExtractor\n",
"tde = TextDescriptivesExtractor(\n",
- " model = \"en\",\n",
- " metrics = None,\n",
- " visible_for_annotators = False,\n",
- " show_progress = True,\n",
+ " model=\"en\",\n",
+ " metrics=None,\n",
+ " visible_for_annotators=False,\n",
+ " show_progress=True,\n",
")"
]
},
@@ -431,10 +432,10 @@
"source": [
"# Initialize the TextDescriptivesExtractor\n",
"tde = TextDescriptivesExtractor(\n",
- " model = \"en\",\n",
- " metrics = [\"descriptive_stats\", \"readability\"],\n",
- " visible_for_annotators = True,\n",
- " show_progress = True,\n",
+ " model=\"en\",\n",
+ " metrics=[\"descriptive_stats\", \"readability\"],\n",
+ " visible_for_annotators=True,\n",
+ " show_progress=True,\n",
")"
]
},
diff --git a/docs/_source/practical_guides/annotation_workflows/weak_supervision.ipynb b/docs/_source/practical_guides/annotation_workflows/weak_supervision.ipynb
index 0370b6a308..b5a0721e35 100644
--- a/docs/_source/practical_guides/annotation_workflows/weak_supervision.ipynb
+++ b/docs/_source/practical_guides/annotation_workflows/weak_supervision.ipynb
@@ -274,7 +274,7 @@
"test_df = pd.read_csv(\"../../tutorials/notebooks/data/yt_comments_test.csv\")\n",
"\n",
"# preview data\n",
- "train_df.head()\n"
+ "train_df.head()"
]
},
{
@@ -369,7 +369,7 @@
"subscribe = Rule(query=\"subscribe\", label=\"SPAM\")\n",
"my = Rule(query=\"my\", label=\"SPAM\")\n",
"song = Rule(query=\"song\", label=\"HAM\")\n",
- "love = Rule(query=\"love\", label=\"HAM\")\n"
+ "love = Rule(query=\"love\", label=\"HAM\")"
]
},
{
@@ -390,6 +390,7 @@
"source": [
"import re\n",
"\n",
+ "\n",
"# rules defined as Python labeling functions\n",
"def contains_http(record: rg.TextClassificationRecord):\n",
" if \"http\" in record.inputs[\"text\"]:\n",
@@ -403,7 +404,7 @@
"def regex_check_out(record: rg.TextClassificationRecord):\n",
" return (\n",
" \"SPAM\" if re.search(r\"check.*out\", record.inputs[\"text\"], flags=re.I) else None\n",
- " )\n"
+ " )"
]
},
{
@@ -422,7 +423,9 @@
"metadata": {},
"outputs": [],
"source": [
- "labeling_rules_df = pd.read_csv(\"../../_static/datasets/weak_supervision_tutorial/labeling_rules.csv\")"
+ "labeling_rules_df = pd.read_csv(\n",
+ " \"../../_static/datasets/weak_supervision_tutorial/labeling_rules.csv\"\n",
+ ")"
]
},
{
@@ -520,9 +523,7 @@
"source": [
"predefined_labeling_rules = []\n",
"for index, row in labeling_rules_df.iterrows():\n",
- " predefined_labeling_rules.append(\n",
- " Rule(row[\"query\"], row[\"label\"])\n",
- " )"
+ " predefined_labeling_rules.append(Rule(row[\"query\"], row[\"label\"]))"
]
},
{
@@ -544,27 +545,16 @@
"from argilla.labeling.text_classification import load_rules, add_rules, delete_rules\n",
"\n",
"# bundle our rules in a list\n",
- "rules = [\n",
- " check_out,\n",
- " plz,\n",
- " subscribe,\n",
- " my,\n",
- " song,\n",
- " love\n",
- "]\n",
+ "rules = [check_out, plz, subscribe, my, song, love]\n",
"\n",
- "labeling_functions = [ \n",
- " contains_http,\n",
- " short_comment,\n",
- " regex_check_out\n",
- "]\n",
+ "labeling_functions = [contains_http, short_comment, regex_check_out]\n",
"\n",
"# add rules to dataset\n",
"add_rules(dataset=\"weak_supervision_yt\", rules=rules)\n",
"\n",
"\n",
"# add the predefined rules loaded from external file\n",
- "add_rules(dataset=\"weak_supervision_yt\", rules=predefined_labeling_rules)\n"
+ "add_rules(dataset=\"weak_supervision_yt\", rules=predefined_labeling_rules)"
]
},
{
@@ -583,7 +573,7 @@
"metadata": {},
"outputs": [],
"source": [
- "# load all the rules available in the dataset including interactively defined in the UI \n",
+ "# load all the rules available in the dataset including interactively defined in the UI\n",
"dataset_labeling_rules = load_rules(dataset=\"weak_supervision_yt\")\n",
"\n",
"# extend the labeling rules with labeling functions\n",
@@ -865,7 +855,7 @@
"not_informative_rules = [\n",
" Rule(\"rich\", \"SPAM\"),\n",
" Rule(\"film\", \"HAM\"),\n",
- " Rule(\"meeting\", \"HAM\")\n",
+ " Rule(\"meeting\", \"HAM\"),\n",
"]"
]
},
@@ -877,6 +867,7 @@
"outputs": [],
"source": [
"from argilla.labeling.text_classification import delete_rules\n",
+ "\n",
"delete_rules(dataset=\"weak_supervision_yt\", rules=not_informative_rules)"
]
},
@@ -1185,7 +1176,7 @@
"from argilla.labeling.text_classification import MajorityVoter\n",
"\n",
"# instantiate the majority vote label model by simply providing the weak labels object\n",
- "majority_model = MajorityVoter(weak_labels)\n"
+ "majority_model = MajorityVoter(weak_labels)"
]
},
{
@@ -1222,7 +1213,7 @@
],
"source": [
"# check its performance\n",
- "print(majority_model.score(output_str=True))\n"
+ "print(majority_model.score(output_str=True))"
]
},
{
@@ -1252,7 +1243,7 @@
"\n",
"# accuracy without abstentions: 0.96; accuracy of random classifier: 0.5\n",
"print(\"accuracy_c:\", frac_non * 0.96 + frac_abs * 0.5)\n",
- "# accuracy_c: 0.868\n"
+ "# accuracy_c: 0.868"
]
},
{
@@ -1295,7 +1286,7 @@
"# extract training data\n",
"training_data = pd.DataFrame(\n",
" [{\"text\": rec.text, \"label\": rec.prediction[0][0]} for rec in records_for_training]\n",
- ")\n"
+ ")"
]
},
{
@@ -1414,7 +1405,7 @@
],
"source": [
"# preview training data\n",
- "training_data\n"
+ "training_data"
]
},
{
@@ -1452,7 +1443,7 @@
"snorkel_model = Snorkel(weak_labels)\n",
"\n",
"# we fit the model\n",
- "snorkel_model.fit(lr=0.001, n_epochs=50)\n"
+ "snorkel_model.fit(lr=0.001, n_epochs=50)"
]
},
{
@@ -1503,7 +1494,7 @@
],
"source": [
"# we check its performance\n",
- "print(snorkel_model.score(output_str=True))\n"
+ "print(snorkel_model.score(output_str=True))"
]
},
{
@@ -1529,7 +1520,7 @@
"\n",
"# accuracy without abstentions: 0.95; accuracy of random classifier: 0.5\n",
"print(\"accuracy_c:\", frac_non * 0.95 + frac_abs * 0.5)\n",
- "# accuracy_c: 0.8761999999999999\n"
+ "# accuracy_c: 0.8761999999999999"
]
},
{
@@ -1560,7 +1551,7 @@
"# extract training data\n",
"training_data = pd.DataFrame(\n",
" [{\"text\": rec.text, \"label\": rec.prediction[0][0]} for rec in records_for_training]\n",
- ")\n"
+ ")"
]
},
{
@@ -1775,7 +1766,7 @@
],
"source": [
"# we check its performance\n",
- "print(flyingsquid_model.score(output_str=True))\n"
+ "print(flyingsquid_model.score(output_str=True))"
]
},
{
@@ -2101,7 +2092,7 @@
" X_test=X_test,\n",
" Y_test=weak_labels.annotation(),\n",
" batch_size=8,\n",
- ")\n"
+ ")"
]
},
{
@@ -2137,7 +2128,7 @@
"trainer.fit(\n",
" model=weasel,\n",
" datamodule=datamodule,\n",
- ")\n"
+ ")"
]
},
{
@@ -2158,7 +2149,7 @@
"outputs": [],
"source": [
"trainer.test()\n",
- "# {'accuracy': 0.94, ...}\n"
+ "# {'accuracy': 0.94, ...}"
]
},
{
@@ -2184,7 +2175,7 @@
"predicted_probs, predicted_label = weasel.predict(tokenizer(text, return_tensors=\"pt\"))\n",
"\n",
"# Map predicted int to label\n",
- "weak_labels.int2label[int(predicted_label)] # HAM\n"
+ "weak_labels.int2label[int(predicted_label)] # HAM"
]
},
{
@@ -2214,7 +2205,7 @@
")\n",
"\n",
"# use pipeline for predictions\n",
- "classifier(text) # [{'label': 'HAM', 'score': 0.6110987663269043}]\n"
+ "classifier(text) # [{'label': 'HAM', 'score': 0.6110987663269043}]"
]
}
],
diff --git a/docs/_source/tutorials/notebooks/deploying-text2text-dvc-explainability.ipynb b/docs/_source/tutorials/notebooks/deploying-text2text-dvc-explainability.ipynb
index b2ec1c9759..afb95fb698 100644
--- a/docs/_source/tutorials/notebooks/deploying-text2text-dvc-explainability.ipynb
+++ b/docs/_source/tutorials/notebooks/deploying-text2text-dvc-explainability.ipynb
@@ -128,10 +128,7 @@
"source": [
"# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
"# Replace api_key if you configured a custom API key\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"admin.apikey\"\n",
- ")"
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"admin.apikey\")"
]
},
{
@@ -154,7 +151,7 @@
"# # Replace api_url with the url to your HF Spaces URL\n",
"# # Replace api_key if you configured a custom API key\n",
"# rg.init(\n",
- "# api_url=\"https://https://[your-owner-name]-[your_space_name].hf.space\", \n",
+ "# api_url=\"https://https://[your-owner-name]-[your_space_name].hf.space\",\n",
"# api_key=\"admin.apikey\",\n",
"# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
"# )"
@@ -198,9 +195,12 @@
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -383,26 +383,26 @@
}
],
"source": [
- "\n",
- "def dataset_backupper(datasets: List[str], duration: int=60*60*24): \n",
+ "def dataset_backupper(datasets: List[str], duration: int = 60 * 60 * 24):\n",
" while True:\n",
" # load datasets and save as .pkl files\n",
- " for dataset_name in datasets: \n",
+ " for dataset_name in datasets:\n",
" ds = rg.load(dataset_name)\n",
" df = ds.to_pandas()\n",
" df.to_pickle(f\"data/{dataset_name}.pkl\")\n",
"\n",
" # get all .pkl files using glob\n",
- " files = glob.glob('data/*.pkl', recursive=True)\n",
- " [os.system(f'dvc add {file}') for file in files]\n",
- " \n",
+ " files = glob.glob(\"data/*.pkl\", recursive=True)\n",
+ " [os.system(f\"dvc add {file}\") for file in files]\n",
+ "\n",
" # push all .pkl.dvc files to github via git push\n",
" os.system(\"dvc push\")\n",
" os.system(\"git commit -m 'update DVC files'\")\n",
" os.system(\"git push\")\n",
- " \n",
+ "\n",
" time.sleep(duration)\n",
- " \n",
+ "\n",
+ "\n",
"dataset_backupper([\"argilla-dvc\"])"
]
},
diff --git a/docs/_source/tutorials/notebooks/deploying-textclassification-colab-activelearning.ipynb b/docs/_source/tutorials/notebooks/deploying-textclassification-colab-activelearning.ipynb
index cfa6236721..133d6e5545 100644
--- a/docs/_source/tutorials/notebooks/deploying-textclassification-colab-activelearning.ipynb
+++ b/docs/_source/tutorials/notebooks/deploying-textclassification-colab-activelearning.ipynb
@@ -67,12 +67,14 @@
"!nvidia-smi\n",
"# info on available ram\n",
"from psutil import virtual_memory\n",
+ "\n",
"ram_gb = virtual_memory().total / 1e9\n",
- "print('\\n\\nYour runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))"
+ "print(\"\\n\\nYour runtime has {:.1f} gigabytes of available RAM\\n\".format(ram_gb))"
]
},
{
"cell_type": "markdown",
+ "id": "7fb27b941602401d91542211134fc71a",
"metadata": {},
"source": [
"### Enable Telemetry\n",
@@ -83,14 +85,18 @@
{
"cell_type": "code",
"execution_count": null,
+ "id": "acae54e37e7d407bbb7b55eff062a284",
"metadata": {},
"outputs": [],
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -147,6 +153,7 @@
"outputs": [],
"source": [
"import time\n",
+ "\n",
"time.sleep(30) # sleeping to give ES time to set up. Otherwise downstream code will bug"
]
},
@@ -236,10 +243,12 @@
"from pyngrok import ngrok, conf\n",
"\n",
"print(\"Enter your authtoken, which can be copied from https://dashboard.ngrok.com/auth\")\n",
- "print(\"You need to create a free ngrok account to get an authtoken. The token looks something like this: ASDO1283YZaDu95vysXYIUXZXYRR_54YfASDIb8cpNfVoz349587\")\n",
+ "print(\n",
+ " \"You need to create a free ngrok account to get an authtoken. The token looks something like this: ASDO1283YZaDu95vysXYIUXZXYRR_54YfASDIb8cpNfVoz349587\"\n",
+ ")\n",
"conf.get_default().auth_token = getpass.getpass()\n",
- "# if the above does not work, you can try: \n",
- "#ngrok.set_auth_token(\"\")"
+ "# if the above does not work, you can try:\n",
+ "# ngrok.set_auth_token(\"\")"
]
},
{
@@ -255,12 +264,18 @@
"[ngrok.disconnect(tunnel.public_url) for tunnel in ngrok.get_tunnels()]\n",
"\n",
"# create the public link\n",
- "# ! check whether this is actually the localhost port Argilla is running on via the terminal above \n",
- "ngrok_tunnel = ngrok.connect(6900) # insert the port number Argilla is running on. e.g. 6900 if the terminal displays something like \"Uvicorn running on http://0.0.0.0:6900\"\n",
- "print(\"You can now access the Argilla localhost with the public link below. (It should look something like 'http://X03b-34-XXX-237-25.ngrok.io')\\n\")\n",
+ "# ! check whether this is actually the localhost port Argilla is running on via the terminal above\n",
+ "ngrok_tunnel = ngrok.connect(\n",
+ " 6900\n",
+ ") # insert the port number Argilla is running on. e.g. 6900 if the terminal displays something like \"Uvicorn running on http://0.0.0.0:6900\"\n",
+ "print(\n",
+ " \"You can now access the Argilla localhost with the public link below. (It should look something like 'http://X03b-34-XXX-237-25.ngrok.io')\\n\"\n",
+ ")\n",
"print(f\"Your ngrok public link: {ngrok_tunnel}\\n\")\n",
"print(\"After clicking on the link, there will be a warning, which you can ignore\")\n",
- "print(\"You can then login with the default argilla username 'argilla' and password '1234'\")"
+ "print(\n",
+ " \"You can then login with the default argilla username 'argilla' and password '1234'\"\n",
+ ")"
]
},
{
@@ -294,10 +309,11 @@
"source": [
"# load dataset\n",
"import datasets\n",
+ "\n",
"dataset_name = \"trec\"\n",
"dataset_hf = datasets.load_dataset(dataset_name, version=datasets.Version(\"2.0.0\"))\n",
"# we work with only a sixth of the texts of the dataset for faster testing\n",
- "dataset_hf[\"train\"] = dataset_hf[\"train\"].shard(num_shards=6, index=0)\n"
+ "dataset_hf[\"train\"] = dataset_hf[\"train\"].shard(num_shards=6, index=0)"
]
},
{
@@ -316,13 +332,17 @@
"# Choose transformer model: In non-gpu environments we use a tiny model to increase efficiency\n",
"if not torch.cuda.is_available():\n",
" transformer_model = \"prajjwal1/bert-tiny\"\n",
- " print(f\"No GPU is available, we therefore use the small model '{transformer_model}' for the active learning loop.\\n\")\n",
+ " print(\n",
+ " f\"No GPU is available, we therefore use the small model '{transformer_model}' for the active learning loop.\\n\"\n",
+ " )\n",
"else:\n",
- " transformer_model = \"microsoft/deberta-v3-xsmall\" #\"bert-base-uncased\"\n",
- " print(f\"A GPU is available, we can therefore use '{transformer_model}' for the active learning loop.\\n\")\n",
+ " transformer_model = \"microsoft/deberta-v3-xsmall\" # \"bert-base-uncased\"\n",
+ " print(\n",
+ " f\"A GPU is available, we can therefore use '{transformer_model}' for the active learning loop.\\n\"\n",
+ " )\n",
"\n",
"# Init tokenizer\n",
- "tokenizer = AutoTokenizer.from_pretrained(transformer_model)\n"
+ "tokenizer = AutoTokenizer.from_pretrained(transformer_model)"
]
},
{
@@ -355,8 +375,7 @@
"\n",
"dataset_test = TransformersDataset.from_arrays(\n",
" test_text, test_labels, tokenizer, target_labels=np.arange(num_classes)\n",
- ")\n",
- "\n"
+ ")"
]
},
{
@@ -384,8 +403,13 @@
"clf_factory = TransformerBasedClassificationFactory(\n",
" TransformerModelArguments(transformer_model),\n",
" num_classes=num_classes,\n",
- " kwargs={\"device\": device, \"num_epochs\": num_epochs, \"lr\": 2e-05, \"mini_batch_size\": 8,\n",
- " \"early_stopping_no_improvement\": 5}\n",
+ " kwargs={\n",
+ " \"device\": device,\n",
+ " \"num_epochs\": num_epochs,\n",
+ " \"lr\": 2e-05,\n",
+ " \"mini_batch_size\": 8,\n",
+ " \"early_stopping_no_improvement\": 5,\n",
+ " },\n",
")\n",
"\n",
"\n",
@@ -393,7 +417,7 @@
"query_strategy = BreakingTies()\n",
"\n",
"# Use the active learner with a pool containing all unlabeled data\n",
- "active_learner = PoolBasedActiveLearner(clf_factory, query_strategy, dataset_st)\n"
+ "active_learner = PoolBasedActiveLearner(clf_factory, query_strategy, dataset_st)"
]
},
{
@@ -407,7 +431,11 @@
"source": [
"## draw an initial sample for the first annotation round\n",
"# https://small-text.readthedocs.io/en/v1.1.1/components/initialization.html\n",
- "from small_text import random_initialization, random_initialization_stratified, random_initialization_balanced\n",
+ "from small_text import (\n",
+ " random_initialization,\n",
+ " random_initialization_stratified,\n",
+ " random_initialization_balanced,\n",
+ ")\n",
"import numpy as np\n",
"\n",
"# Fix seed for reproducibility\n",
@@ -417,9 +445,9 @@
"NUM_SAMPLES = 10\n",
"\n",
"# Draw an initial subset from the data pool\n",
- "#initial_indices = random_initialization(dataset_st, NUM_SAMPLES)\n",
- "#initial_indices = random_initialization_balanced(train_labels, NUM_SAMPLES)\n",
- "initial_indices = random_initialization_stratified(train_labels, NUM_SAMPLES)\n"
+ "# initial_indices = random_initialization(dataset_st, NUM_SAMPLES)\n",
+ "# initial_indices = random_initialization_balanced(train_labels, NUM_SAMPLES)\n",
+ "initial_indices = random_initialization_stratified(train_labels, NUM_SAMPLES)"
]
},
{
@@ -455,7 +483,7 @@
"]\n",
"\n",
"# Log initial records to Argilla\n",
- "rg.log(records, DATASET_NAME)\n"
+ "rg.log(records, DATASET_NAME)"
]
},
{
@@ -475,6 +503,7 @@
"LABEL2INT = dataset_hf[\"train\"].features[\"coarse_label\"].str2int\n",
"ACCURACIES = []\n",
"\n",
+ "\n",
"# Set up the active learning loop with the listener decorator\n",
"@listener(\n",
" dataset=DATASET_NAME,\n",
@@ -527,8 +556,7 @@
" print(\"Waiting for annotations ...\")\n",
"\n",
"\n",
- "\n",
- "active_learning_loop.start()\n"
+ "active_learning_loop.start()"
]
},
{
@@ -564,7 +592,7 @@
],
"source": [
"print(f\"You can now start annotating with active learning in the background!\")\n",
- "print(f\"The public link for accessing the annotation interface is: {ngrok_tunnel}\")\n"
+ "print(f\"The public link for accessing the annotation interface is: {ngrok_tunnel}\")"
]
},
{
@@ -603,7 +631,8 @@
"source": [
"# plot learning progress over different active learning iterations\n",
"import pandas as pd\n",
- "pd.Series(ACCURACIES).plot(xlabel=\"Iteration\", ylabel=\"Accuracy\")\n"
+ "\n",
+ "pd.Series(ACCURACIES).plot(xlabel=\"Iteration\", ylabel=\"Accuracy\")"
]
},
{
diff --git a/docs/_source/tutorials/notebooks/deploying-texttokenclassification-fastapi.ipynb b/docs/_source/tutorials/notebooks/deploying-texttokenclassification-fastapi.ipynb
index 7d1d573230..c03057a4db 100644
--- a/docs/_source/tutorials/notebooks/deploying-texttokenclassification-fastapi.ipynb
+++ b/docs/_source/tutorials/notebooks/deploying-texttokenclassification-fastapi.ipynb
@@ -129,10 +129,7 @@
"source": [
"# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
"# Replace api_key if you configured a custom API key\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"admin.apikey\"\n",
- ")"
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"admin.apikey\")"
]
},
{
@@ -155,7 +152,7 @@
"# # Replace api_url with the url to your HF Spaces URL\n",
"# # Replace api_key if you configured a custom API key\n",
"# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
"# api_key=\"admin.apikey\",\n",
"# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
"# )"
@@ -186,7 +183,7 @@
"\n",
"# for adding logging to API endpoints\n",
"from argilla.monitoring.asgi import (\n",
- " ArgillaLogHTTPMiddleware, \n",
+ " ArgillaLogHTTPMiddleware,\n",
" text_classification_mapper,\n",
" token_classification_mapper,\n",
")\n",
@@ -212,9 +209,12 @@
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -435,11 +435,12 @@
" for data, prediction in zip(batch, outputs)\n",
" ]\n",
"\n",
+ "\n",
"app.add_middleware(\n",
" ArgillaLogHTTPMiddleware,\n",
" api_endpoint=\"/transformers/\", # the endpoint that will be logged\n",
" dataset=\"monitoring_transformers\", # your dataset name\n",
- " records_mapper=text2records, # your post-process func to adapt service inputs and outputs into an Argilla record\n",
+ " records_mapper=text2records, # your post-process func to adapt service inputs and outputs into an Argilla record\n",
")"
]
},
@@ -467,6 +468,7 @@
" for data, prediction in zip(batch, outputs)\n",
" ]\n",
"\n",
+ "\n",
"app.add_middleware(\n",
" ArgillaLogHTTPMiddleware,\n",
" api_endpoint=\"/spacy/\",\n",
@@ -474,6 +476,7 @@
" records_mapper=token2records,\n",
")\n",
"\n",
+ "\n",
"# prediction endpoint using spacy pipeline\n",
"@app.post(\"/ner/\")\n",
"def predict_spacy(batch: List[str]):\n",
@@ -586,8 +589,8 @@
"import requests\n",
"\n",
"response = requests.post(\n",
- " \"http://localhost:8000/sentiment/\", \n",
- " json=[\"I like Argilla\", \"I hated data labelling but now I don't\"]\n",
+ " \"http://localhost:8000/sentiment/\",\n",
+ " json=[\"I like Argilla\", \"I hated data labelling but now I don't\"],\n",
")\n",
"\n",
"response.content"
@@ -634,7 +637,7 @@
"from transformers import pipeline\n",
"\n",
"from argilla.monitoring.asgi import (\n",
- " ArgillaLogHTTPMiddleware, \n",
+ " ArgillaLogHTTPMiddleware,\n",
" text_classification_mapper,\n",
" token_classification_mapper,\n",
")\n",
@@ -644,6 +647,7 @@
"\n",
"app = FastAPI()\n",
"\n",
+ "\n",
"# prediction endpoint using transformers pipeline\n",
"@app.post(\"/sentiment/\")\n",
"def predict_transformers(batch: List[str]):\n",
@@ -656,25 +660,29 @@
" for prediction in predictions\n",
" ]\n",
"\n",
+ "\n",
"def text2records(batch: List[str], outputs: List[dict]):\n",
" return [\n",
" text_classification_mapper(data, prediction)\n",
" for data, prediction in zip(batch, outputs)\n",
" ]\n",
"\n",
+ "\n",
"app.add_middleware(\n",
" ArgillaLogHTTPMiddleware,\n",
" api_endpoint=\"/transformers/\", # the endpoint that will be logged\n",
" dataset=\"monitoring_transformers\", # your dataset name\n",
- " records_mapper=text2records, # your post-process func to adapt service inputs and outputs into an Argilla record\n",
+ " records_mapper=text2records, # your post-process func to adapt service inputs and outputs into an Argilla record\n",
")\n",
"\n",
+ "\n",
"def token2records(batch: List[str], outputs: List[dict]):\n",
" return [\n",
" token_classification_mapper(data, prediction)\n",
" for data, prediction in zip(batch, outputs)\n",
" ]\n",
"\n",
+ "\n",
"# prediction endpoint using spacy pipeline\n",
"@app.post(\"/ner/\")\n",
"def predict_spacy(batch: List[str]):\n",
@@ -694,6 +702,7 @@
" predictions.append(prediction)\n",
" return predictions\n",
"\n",
+ "\n",
"app.add_middleware(\n",
" ArgillaLogHTTPMiddleware,\n",
" api_endpoint=\"/ner/\",\n",
@@ -708,14 +717,13 @@
" records_mapper=text2records,\n",
")\n",
"\n",
+ "\n",
"@app.get(\"/\")\n",
"def root():\n",
" return {\"message\": \"alive\"}\n",
"\n",
- "argilla.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"admin.apikey\"\n",
- ")"
+ "\n",
+ "argilla.init(api_url=\"http://localhost:6900\", api_key=\"admin.apikey\")"
]
}
],
diff --git a/docs/_source/tutorials/notebooks/labelling-spacy-llm.ipynb b/docs/_source/tutorials/notebooks/labelling-spacy-llm.ipynb
index a1ece5425a..c6682dce90 100644
--- a/docs/_source/tutorials/notebooks/labelling-spacy-llm.ipynb
+++ b/docs/_source/tutorials/notebooks/labelling-spacy-llm.ipynb
@@ -108,10 +108,7 @@
"source": [
"# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
"# Replace api_key if you configured a custom API key\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\",\n",
- " api_key=\"admin.apikey\"\n",
- ")"
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"admin.apikey\")"
]
},
{
@@ -134,7 +131,7 @@
"# # Replace api_url with the url to your HF Spaces URL\n",
"# # Replace api_key if you configured a custom API key\n",
"# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
"# api_key=\"admin.apikey\",\n",
"# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
"# )"
@@ -157,9 +154,12 @@
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -247,8 +247,8 @@
"config = configparser.ConfigParser()\n",
"config.read_string(config_string)\n",
"\n",
- "with open(\"config.cfg\", 'w') as configfile:\n",
- " config.write(configfile)"
+ "with open(\"config.cfg\", \"w\") as configfile:\n",
+ " config.write(configfile)"
]
},
{
@@ -299,20 +299,21 @@
"metadata": {},
"outputs": [],
"source": [
- "#returns the category with the highest score\n",
+ "# returns the category with the highest score\n",
"def get_textcat_suggestion(doc):\n",
- " model_prediction = doc.cats\n",
- " return max(model_prediction, key=model_prediction.get)\n",
+ " model_prediction = doc.cats\n",
+ " return max(model_prediction, key=model_prediction.get)\n",
+ "\n",
"\n",
- "#selects the top N sentences with the highest scores and return combined string\n",
+ "# selects the top N sentences with the highest scores and return combined string\n",
"def get_summarization_suggestion(doc):\n",
- " sentence_scores = Counter()\n",
- " for sentence in doc.sents:\n",
- " for word in sentence:\n",
- " sentence_scores[sentence] += 1\n",
- " summary_sentences = nlargest(2, sentence_scores, key=sentence_scores.get)\n",
- " summary = ' '.join(str(sentence) for sentence in summary_sentences)\n",
- " return summary"
+ " sentence_scores = Counter()\n",
+ " for sentence in doc.sents:\n",
+ " for word in sentence:\n",
+ " sentence_scores[sentence] += 1\n",
+ " summary_sentences = nlargest(2, sentence_scores, key=sentence_scores.get)\n",
+ " summary = \" \".join(str(sentence) for sentence in summary_sentences)\n",
+ " return summary"
]
},
{
@@ -354,21 +355,16 @@
"outputs": [],
"source": [
"dataset = rg.FeedbackDataset(\n",
- " fields=[\n",
- " rg.TextField(name=\"text\")\n",
- " ],\n",
+ " fields=[rg.TextField(name=\"text\")],\n",
" questions=[\n",
" rg.LabelQuestion(\n",
" name=\"label-question\",\n",
" title=\"Classify the text category.\",\n",
- " #make sure that the labels are in line with the labels we have defined in config.cfg\n",
- " labels=[\"HISTORY\",\"MUSIC\",\"TECHNOLOGY\",\"SCIENCE\",\"SPORTS\",\"POLITICS\"]\n",
+ " # make sure that the labels are in line with the labels we have defined in config.cfg\n",
+ " labels=[\"HISTORY\", \"MUSIC\", \"TECHNOLOGY\", \"SCIENCE\", \"SPORTS\", \"POLITICS\"],\n",
" ),\n",
- " rg.TextQuestion(\n",
- " name=\"text-question\",\n",
- " title=\"Provide a summary for the text.\"\n",
- " )\n",
- " ]\n",
+ " rg.TextQuestion(name=\"text-question\", title=\"Provide a summary for the text.\"),\n",
+ " ],\n",
")"
]
},
@@ -387,16 +383,16 @@
"source": [
"records = [\n",
" rg.FeedbackRecord(\n",
- " fields={\n",
- " \"text\": doc.text\n",
- " },\n",
+ " fields={\"text\": doc.text},\n",
" suggestions=[\n",
- " {\"question_name\": \"label-question\",\n",
- " \"value\": get_textcat_suggestion(doc)},\n",
- " {\"question_name\":\"text-question\",\n",
- " \"value\": get_summarization_suggestion(doc)}\n",
- " ]\n",
- " ) for doc in [nlp(item) for item in dataset_hf[\"context\"]]\n",
+ " {\"question_name\": \"label-question\", \"value\": get_textcat_suggestion(doc)},\n",
+ " {\n",
+ " \"question_name\": \"text-question\",\n",
+ " \"value\": get_summarization_suggestion(doc),\n",
+ " },\n",
+ " ],\n",
+ " )\n",
+ " for doc in [nlp(item) for item in dataset_hf[\"context\"]]\n",
"]"
]
},
diff --git a/docs/_source/tutorials/notebooks/labelling-text2text-disaggregators-explainability.ipynb b/docs/_source/tutorials/notebooks/labelling-text2text-disaggregators-explainability.ipynb
index 342b0e43c3..f66a5d254c 100644
--- a/docs/_source/tutorials/notebooks/labelling-text2text-disaggregators-explainability.ipynb
+++ b/docs/_source/tutorials/notebooks/labelling-text2text-disaggregators-explainability.ipynb
@@ -120,10 +120,7 @@
"source": [
"# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
"# Replace api_key if you configured a custom API key\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"admin.apikey\"\n",
- ")"
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"admin.apikey\")"
]
},
{
@@ -146,7 +143,7 @@
"# # Replace api_url with the url to your HF Spaces URL\n",
"# # Replace api_key if you configured a custom API key\n",
"# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
"# api_key=\"admin.apikey\",\n",
"# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
"# )"
@@ -190,9 +187,12 @@
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -216,7 +216,9 @@
"dataset_rg = rg.read_datasets(my_dataset[\"train\"], task=\"Text2Text\")\n",
"\n",
"# log subset into argilla\n",
- "rg.log(dataset_rg[:1000], \"news-summary\", chunk_size=50) # set smaller chunk size to overcome io-issues"
+ "rg.log(\n",
+ " dataset_rg[:1000], \"news-summary\", chunk_size=50\n",
+ ") # set smaller chunk size to overcome io-issues"
]
},
{
@@ -375,7 +377,7 @@
"metadata_ds = df[df.columns[1:]].to_dict(orient=\"records\")\n",
"for metadata_rec, rec in zip(metadata_ds, ds):\n",
" rec.metadata = metadata_rec\n",
- "rg.log(ds, \"news-summary\", chunk_size=50) # upsert records"
+ "rg.log(ds, \"news-summary\", chunk_size=50) # upsert records"
]
},
{
diff --git a/docs/_source/tutorials/notebooks/labelling-textclassification-gpt3-fewshot.ipynb b/docs/_source/tutorials/notebooks/labelling-textclassification-gpt3-fewshot.ipynb
index 379c43915b..253bd5348c 100644
--- a/docs/_source/tutorials/notebooks/labelling-textclassification-gpt3-fewshot.ipynb
+++ b/docs/_source/tutorials/notebooks/labelling-textclassification-gpt3-fewshot.ipynb
@@ -115,14 +115,12 @@
"source": [
"# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
"# Replace api_key if you configured a custom API key\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"admin.apikey\"\n",
- ")"
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"admin.apikey\")"
]
},
{
"cell_type": "markdown",
+ "id": "7fb27b941602401d91542211134fc71a",
"metadata": {},
"source": [
"If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
@@ -131,6 +129,7 @@
{
"cell_type": "code",
"execution_count": null,
+ "id": "acae54e37e7d407bbb7b55eff062a284",
"metadata": {},
"outputs": [],
"source": [
@@ -141,7 +140,7 @@
"# # Replace api_url with the url to your HF Spaces URL\n",
"# # Replace api_key if you configured a custom API key\n",
"# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
"# api_key=\"admin.apikey\",\n",
"# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
"# )"
@@ -176,6 +175,7 @@
},
{
"cell_type": "markdown",
+ "id": "9a63283cbaf04dbcab1f6479b197f3a8",
"metadata": {},
"source": [
"### Enable Telemetry\n",
@@ -186,14 +186,18 @@
{
"cell_type": "code",
"execution_count": null,
+ "id": "8dd0d8092fe74a7c96281538738b07e2",
"metadata": {},
"outputs": [],
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -445,17 +449,15 @@
"outputs": [],
"source": [
"# set your api key as ENV, for example with Python: os.environ[\"OPENAI_API_KEY\"]\n",
- "openai.api_key = os.getenv(\"OPENAI_API_KEY\") \n",
+ "openai.api_key = os.getenv(\"OPENAI_API_KEY\")\n",
+ "\n",
"\n",
"def classify(text):\n",
" # build prompt with template and input\n",
" prompt = f\"{PROMPT_TEMPLATE}\\n{text}\\n\"\n",
" # use create completion template\n",
" completion = openai.Completion.create(\n",
- " model=\"text-davinci-003\",\n",
- " prompt=prompt,\n",
- " temperature=0,\n",
- " max_tokens=256\n",
+ " model=\"text-davinci-003\", prompt=prompt, temperature=0, max_tokens=256\n",
" )\n",
" # get first choice text\n",
" json_response = completion[\"choices\"][0][\"text\"].strip()\n",
@@ -463,8 +465,11 @@
" prediction = loads(json_response)\n",
" except:\n",
" # for some examples, json is not correctly formatted\n",
- " return {\"prediction\": None, \"explanation\": f\"Wrong JSON format: {json_response}\" }\n",
- " return prediction "
+ " return {\n",
+ " \"prediction\": None,\n",
+ " \"explanation\": f\"Wrong JSON format: {json_response}\",\n",
+ " }\n",
+ " return prediction"
]
},
{
@@ -682,7 +687,7 @@
"# let's predict over the test set to eval our zero-shot classifier\n",
"test_ds_with_preds = banking_ds[\"test\"].map(lambda example: classify(example[\"text\"]))\n",
"\n",
- "pd.set_option('display.max_colwidth', None)\n",
+ "pd.set_option(\"display.max_colwidth\", None)\n",
"test_ds_with_preds.to_pandas().head(15)"
]
},
@@ -716,7 +721,7 @@
" record = rg.TextClassificationRecord(\n",
" inputs={\"text\": example[\"text\"], \"explanation\": example[\"explanation\"]},\n",
" annotation=labels[example[\"label\"]],\n",
- " prediction=[(example[\"prediction\"].lower(), 1.0)]\n",
+ " prediction=[(example[\"prediction\"].lower(), 1.0)],\n",
" )\n",
" records.append(record)\n",
"\n",
@@ -1745,7 +1750,7 @@
],
"range": [
0,
- 0.9090909090909091
+ 0.9090909090909092
],
"type": "linear"
}
@@ -5173,7 +5178,7 @@
],
"range": [
0,
- 0.9090909090909091
+ 0.9090909090909092
],
"type": "linear"
}
diff --git a/docs/_source/tutorials/notebooks/labelling-textclassification-sentence-transformers-semantic.ipynb b/docs/_source/tutorials/notebooks/labelling-textclassification-sentence-transformers-semantic.ipynb
index 6281388bf5..3e2c1b7058 100644
--- a/docs/_source/tutorials/notebooks/labelling-textclassification-sentence-transformers-semantic.ipynb
+++ b/docs/_source/tutorials/notebooks/labelling-textclassification-sentence-transformers-semantic.ipynb
@@ -116,10 +116,7 @@
"source": [
"# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
"# Replace api_key if you configured a custom API key\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"admin.apikey\"\n",
- ")"
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"admin.apikey\")"
]
},
{
@@ -142,7 +139,7 @@
"# # Replace api_url with the url to your HF Spaces URL\n",
"# # Replace api_key if you configured a custom API key\n",
"# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
"# api_key=\"admin.apikey\",\n",
"# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
"# )"
@@ -162,7 +159,6 @@
"metadata": {},
"outputs": [],
"source": [
- "\n",
"from sentence_transformers import SentenceTransformer\n",
"from datasets import load_dataset"
]
@@ -184,9 +180,12 @@
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -213,9 +212,9 @@
"\n",
"# Encode text field using batched computation\n",
"dataset = dataset.map(\n",
- " lambda batch: {\"vectors\": encoder.encode(batch[\"text\"])}, \n",
- " batch_size=32, \n",
- " batched=True\n",
+ " lambda batch: {\"vectors\": encoder.encode(batch[\"text\"])},\n",
+ " batch_size=32,\n",
+ " batched=True,\n",
")\n",
"\n",
"# Removes the original labels because you'll be labeling from scratch\n",
@@ -342,7 +341,16 @@
"\n",
"# Our labeling scheme\n",
"settings = rg.TextClassificationSettings(\n",
- " label_schema=[\"change_details\", \"card\", \"atm\", \"top_up\", \"balance\", \"transfer\", \"exchange_rate\", \"pin\"]\n",
+ " label_schema=[\n",
+ " \"change_details\",\n",
+ " \"card\",\n",
+ " \"atm\",\n",
+ " \"top_up\",\n",
+ " \"balance\",\n",
+ " \"transfer\",\n",
+ " \"exchange_rate\",\n",
+ " \"pin\",\n",
+ " ]\n",
")\n",
"\n",
"rg.configure_dataset_settings(name=\"banking77-topics\", settings=settings)\n",
diff --git a/docs/_source/tutorials/notebooks/labelling-textclassification-sentencetransformers-semantic.ipynb b/docs/_source/tutorials/notebooks/labelling-textclassification-sentencetransformers-semantic.ipynb
index 144af598c6..05c2a28437 100644
--- a/docs/_source/tutorials/notebooks/labelling-textclassification-sentencetransformers-semantic.ipynb
+++ b/docs/_source/tutorials/notebooks/labelling-textclassification-sentencetransformers-semantic.ipynb
@@ -1,10378 +1,10395 @@
{
- "cells": [
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "5PwXNf1WNYm4"
- },
- "source": [
- "# 📸 Bulk Labeling Multimodal Data\n",
- "\n",
- "\n",
- "In this tutorial, we will work with multimodal data of images and text. It will walk you through the following steps:\n",
- "\n",
- "- Load a dataset with images and text of electronic products.\n",
- "- Experiment with zero-shot image and text classification.\n",
- "- Label the data using bulk labelling with image and text embeddings.\n",
- "- Train a SetFit classification model on the labelled data.\n",
- "\n",
- "\n"
- ]
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "5PwXNf1WNYm4"
+ },
+ "source": [
+ "# 📸 Bulk Labeling Multimodal Data\n",
+ "\n",
+ "\n",
+ "In this tutorial, we will work with multimodal data of images and text. It will walk you through the following steps:\n",
+ "\n",
+ "- Load a dataset with images and text of electronic products.\n",
+ "- Experiment with zero-shot image and text classification.\n",
+ "- Label the data using bulk labelling with image and text embeddings.\n",
+ "- Train a SetFit classification model on the labelled data.\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Running Argilla\n",
+ "\n",
+ "For this tutorial, you will need to have an Argilla server running. There are two main options for deploying and running Argilla:\n",
+ "\n",
+ "1. [Deploy Argilla on Hugging Face Spaces](https://huggingface.co/docs/hub/spaces-sdks-docker-argilla): This is the fastest option and the recommended choice for connecting to external notebooks (e.g., Google Colab) if you have an account on Hugging Face.\n",
+ "\n",
+ "2. [Launch Argilla using Argilla's quickstart Docker image](../../getting_started/quickstart.ipynb): This is the recommended option if you want Argilla running on your local machine. Note that this option will only let you run the tutorial locally and not with an external notebook service.\n",
+ "\n",
+ "For more information on deployment options, please check the Deployment section of the documentation.\n",
+ "\n",
+ "
\n",
+ "\n",
+ "Tip\n",
+ " \n",
+ "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
+ "\n",
+ "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
+ "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
+ "
\n",
+ "\n",
+ "## Setup\n",
+ "\n",
+ "For this tutorial, you'll need to install the Argilla client and a few third-party libraries using `pip`:"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "hCR_hYiWNYm7"
+ },
+ "source": [
+ "## Introduction\n",
+ "\n",
+ "__Real-world multimodal data__ is often a mix of text and images. In this tutorial, we will work with a dataset of electronic products. The dataset contains images of the products and a description of the product. \n",
+ "\n",
+ "This notebook uses a dataset of electronics parts and products from a fictional electronics webshop. \n",
+ "\n",
+ "Let's get started!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "id": "B-DDXAVxNYm7"
+ },
+ "outputs": [],
+ "source": [
+ "%pip install argilla \"setfit~=0.2.0\" \"datasets~=2.3.0\" transformers sentence-transformers -qqq"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "le5C4pzTNYm9"
+ },
+ "source": [
+ "Let's import the Argilla module for reading and writing data:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "id": "KNr0WEVWNYm-"
+ },
+ "outputs": [],
+ "source": [
+ "import argilla as rg"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "J5ecm_MjNYm-"
+ },
+ "source": [
+ "If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "fEDgKhSVNYm-"
+ },
+ "outputs": [],
+ "source": [
+ "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
+ "# Replace api_key if you configured a custom API key\n",
+ "rg.init(api_url=\"https://localhost:6900\", api_key=\"admin.apikey\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Set the HF_TOKEN environment variable\n",
+ "# import os\n",
+ "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
+ "\n",
+ "# # Replace api_url with the url to your HF Spaces URL\n",
+ "# # Replace api_key if you configured a custom API key\n",
+ "# rg.init(\n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
+ "# api_key=\"admin.apikey\",\n",
+ "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
+ "# )"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "VgNib6EDNYm-"
+ },
+ "source": [
+ "Finally, let's include the imports we need:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import pprint as pp\n",
+ "from requests import get\n",
+ "\n",
+ "from datasets import load_dataset\n",
+ "from PIL import Image\n",
+ "from sklearn.metrics import accuracy_score\n",
+ "from sentence_transformers import SentenceTransformer\n",
+ "from transformers import pipeline\n",
+ "from sentence_transformers.losses import CosineSimilarityLoss\n",
+ "from setfit import SetFitModel, SetFitTrainer\n",
+ "from PIL import Image"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Enable Telemetry\n",
+ "\n",
+ "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
+ " tutorial_running()\n",
+ "except ImportError:\n",
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "izZIHEzYNYnB"
+ },
+ "source": [
+ "## A 'real-world' multimodal dataset\n",
+ "\n",
+ "The dataset samples contain a `page_name`, `page_descriptions`, and a `label`. The dataset is split into two parts: `labelled` and `unlabelled`. The labelled portion is the result of my annotation so we can test methods. In reality, let's say this doesn't exist 😏. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 438,
+ "referenced_widgets": [
+ "d2cf4d1133a2421aa1ec980bdca4fecd",
+ "bcfc02b30e024b5c81856bcd9e191d07",
+ "eaefc3e261a7453cb66905079b71f16a",
+ "ac546920996e4ca081e438999ae19fc8",
+ "c54e39be37434e0d826d86041ef1de96",
+ "f4d6914361f44912b0d7ca1b16496606",
+ "b3c787a117ae42738c315c5cdabd508c",
+ "edf41d1718984e7dbef2b65799935fd8",
+ "20dad493a55343efaabbc6f7c1966cf0",
+ "b91ae0e5a43143e68f53099e85f1afc2",
+ "a1ec5941f87f4b23bc72188313ac9275",
+ "6482c21eb5d449f795a9e39fcae6a46e",
+ "2db4287290eb4c25bc440fc3f2f1d258",
+ "b470309f2a6b4878852ae9e589764edf",
+ "63de802f18e44b16a3a2f5029ad7b705",
+ "670f91d2101c4d87997c228210646f89",
+ "d8f11947b0d048489bdd649479c50b1f",
+ "70eee61c4d3842f69569dee3ac8773fa",
+ "26175752e18140b4a944bfc55f77a069",
+ "c13ea0c47f6e48f0a804fa9f2f5ce544",
+ "9e17b92da1aa43d0aa980db275437fde",
+ "d2186c7599164e528aa3d92f6999a9bb",
+ "35f1224563044be39e254342c6c5b635",
+ "87a472fb7193452faea139ab6bf1c9be",
+ "eb9b58dab8044074b1cac3e39f26e561",
+ "6359430089654ca4bdd587c125b317a0",
+ "be06bfcbe41040369d025626b8227b6b",
+ "082dfef575c94327a5377c3f7278cf5a",
+ "84d1a26c298b42cd820bcd2e4302a443",
+ "0987a64b9cea4850998e78d76d31c3b8",
+ "fa0573f2ebd84a5ba5ebdc0452362887",
+ "e8e295d7292245fb9f17413599d59960",
+ "0286aff9c56b43e7abd15f17dd33d6db",
+ "8b161e12449f4078bb64245518bb596d",
+ "71181e8f913b412fb72917c55a151723",
+ "dcd3ba169df34a0395eb5d03405f8835",
+ "d0b05902a4db4abd8c10cab6178a59ec",
+ "8294163893bc40c2bef26032babb3938",
+ "735a35f35dee4d548cd6284f9e3e1f5a",
+ "b7dfa3f0d2374482af6dc25ec8dd65df",
+ "c65f5b1c90e24d47a060c0b362749b82",
+ "01bc8315e7cd4064a8653eae44cea192",
+ "36156c14c8d544deab0ea65ac5ba7508",
+ "15d85f4fe15545bc85d3717d7af7bbb1",
+ "f8d66c452a6d4347b9a8247ad135fd35",
+ "3c8aa1d02f1541bdb59b0a8dff27148f",
+ "88677a526eae40a1b5e0b5e9e7a25218",
+ "eaf5ad9e9e3240799128ef8bbf0a195a",
+ "48f63d3af7604e8abe4ca94dbae63f48",
+ "27d3f1981c244549901253b3266de1ff",
+ "9aae9636296a4245989388403103e0eb",
+ "139f9e8c84f447b985c2080c17d14a8c",
+ "1d37c4be099b44b5bc67c2cd976693b9",
+ "1ad0c7637293497a9a75bfa45c1a06e8",
+ "4fc202bfa55842b48e59d4a800026331",
+ "bbb9442a3ec1463790bcf6533f3be905",
+ "16bd920f956f43b3921b79cec4d76d87",
+ "c12bc29016124339a6c3cb11f21bfaf0",
+ "a6b61668ad1146e78c2a9b2556b23703",
+ "14cc3d3e93c54226bc2f38a4745c1d23",
+ "16fccf90f2974a938167677b189b67a2",
+ "714590bcc27e4efc837f870812929f3e",
+ "9cef8cafa31a47fd9f7410478b5415e8",
+ "2e4deb55d7b74e2fa09c9f925fd2e751",
+ "2433cd897d734c78b3d4a62602865415",
+ "9fac34a2d3e44c95a2591b26ddae5c1e",
+ "88edaefb70c34854ad6f16d5a82b40e7",
+ "ba93d6edfed5434397d869ba2bd7aca5",
+ "7282d9ef9a754bcfb1297081296aed5a",
+ "946c67b0a3f54ce09c35db7bef44991c",
+ "f41cf420defb4b109f836fecea8e1d15",
+ "c10b310f7f6344b39c8d165c887d28f1",
+ "052b62b792fc4f1183ffb91f18d655c7",
+ "c4a54f5262014b77964d42429d5e43bf",
+ "cf540757bc79428bab170e157eb9381f",
+ "03390d37eba24242ae6c2099d5fe7863",
+ "88901ee5ca86458288bf82a22db7e379",
+ "498becb6513a44a4a17cff76a8ae4666",
+ "5066a8fd18d94f9da0aa0cf354d13452",
+ "a8d41a62f1b94575a744c12a399de8c8",
+ "e8cc8dbe0d164a9eaec472ffe1351db9",
+ "0d3cbcf01ca14063be3ae129db78fdcb",
+ "1a951db095d441e295bb9ff00d4dba30",
+ "e120b072886b4d4f94ef506ca3d6a605",
+ "d5fea5e89ebd4d298c169526eadeeb32",
+ "8513e53c07c04d75999be1fec6d5b491",
+ "e7da6c65cec140c3801fca68bcfdebcc",
+ "eccef3d9147242d9a1e42f4abd0ebe4c",
+ "b77605d5afb94b8cb579f3c690ea4203",
+ "aedab95f1bb144c38180f2b2e71c14ff",
+ "49d11634ca864c04bc87e472d3563e73",
+ "20163aeba069412eabd3b506a339667f",
+ "feef17ea90ae4105b2ffb56841ed2adb",
+ "e97c5c6e969842468f53520dfba1ee8f",
+ "89b04536243a48049eb777f00fe49555",
+ "94502dbbc1ab42f48d2d375a8265dbb4",
+ "090c3e60ff0d4e5fa930767672802d86",
+ "b577441dde9e43cb9644244ca0cf336e",
+ "b850f11900ae4874963b35e0b86470e5",
+ "963fbb3d5b9f4848a8cadf2a2d2b264b",
+ "52ad436eea8649058457d6380f618ea2",
+ "5817239aa98046e9be3736688f8c1a5f",
+ "c48e02dc87d74d50ba6de4aefeb1441e",
+ "ecc225aab6804d97b1cdd383621c9695",
+ "fcfda905fd4448b3940b04f4136c53a4",
+ "16162c715d4745f3aa91c7f38a78a32d",
+ "4c7287d821f1471e9ebdafa2f8229d28",
+ "5a65206e09c04176bfd33bcb31cd1ad5",
+ "9ac85729093c41c1b2a0fef5191b8ab5",
+ "d774f350171b4c3fb6be768dbf11c726"
+ ]
},
+ "id": "gdVdxJt-NYnC",
+ "outputId": "c54aa613-806e-42fa-b4de-f374c78b1d50"
+ },
+ "outputs": [],
+ "source": [
+ "ELECTRONICS_DATASET = \"burtenshaw/electronics\"\n",
+ "dataset = load_dataset(ELECTRONICS_DATASET)\n",
+ "labels = dataset[\"labelled\"].features[\"label\"].names\n",
+ "int2str = dataset[\"labelled\"].features[\"label\"].int2str"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
{
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Running Argilla\n",
- "\n",
- "For this tutorial, you will need to have an Argilla server running. There are two main options for deploying and running Argilla:\n",
- "\n",
- "1. [Deploy Argilla on Hugging Face Spaces](https://huggingface.co/docs/hub/spaces-sdks-docker-argilla): This is the fastest option and the recommended choice for connecting to external notebooks (e.g., Google Colab) if you have an account on Hugging Face.\n",
- "\n",
- "2. [Launch Argilla using Argilla's quickstart Docker image](../../getting_started/quickstart.ipynb): This is the recommended option if you want Argilla running on your local machine. Note that this option will only let you run the tutorial locally and not with an external notebook service.\n",
- "\n",
- "For more information on deployment options, please check the Deployment section of the documentation.\n",
- "\n",
- "
\n",
- "\n",
- "Tip\n",
- " \n",
- "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
- "\n",
- "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
- "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
- "
\n",
- "\n",
- "## Setup\n",
- "\n",
- "For this tutorial, you'll need to install the Argilla client and a few third-party libraries using `pip`:"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'image_url': 'https://tse1.mm.bing.net/th?id=OIP.to3Cddhws6ECl-_ySZ5ShQHaFi&pid=Api',\n",
+ " 'label': 1,\n",
+ " 'page_description': '\\n'\n",
+ " '\\n'\n",
+ " 'Are you looking for a way to reduce the number of '\n",
+ " 'purchase orders you need to place for cable assemblies? '\n",
+ " \"If so, then this guide is for you! We'll show you how to \"\n",
+ " 'source cable assemblies with fewer purchase orders, '\n",
+ " \"saving you time and money. We'll cover topics such as \"\n",
+ " 'understanding the different types of cable assemblies, '\n",
+ " 'researching suppliers, and negotiating the best prices. '\n",
+ " \"We'll also provide tips on how to streamline the \"\n",
+ " 'ordering process and ensure you get the best quality '\n",
+ " \"products. With this guide, you'll be able to source \"\n",
+ " 'cable assemblies with fewer purchase orders and get the '\n",
+ " 'most out of your budget.',\n",
+ " 'page_name': 'How to Source Cable Assemblies With Fewer Purchase Orders ...'}\n"
+ ]
+ }
+ ],
+ "source": [
+ "# show a sample\n",
+ "pp.pprint(next(iter(dataset[\"labelled\"])))"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 🔫 Zero-Shot Classification"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "U9R2eaRXNYnC"
+ },
+ "source": [
+ "### 📷 Images\n",
+ "\n",
+ "First, we will explore some zero-shot techniques. For the sake of comparison, we will use the `labelled` portion of the dataset.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "hCR_hYiWNYm7"
- },
- "source": [
- "## Introduction\n",
- "\n",
- "__Real-world multimodal data__ is often a mix of text and images. In this tutorial, we will work with a dataset of electronic products. The dataset contains images of the products and a description of the product. \n",
- "\n",
- "This notebook uses a dataset of electronics parts and products from a fictional electronics webshop. \n",
- "\n",
- "Let's get started!"
- ]
+ "id": "SshZkW1eNYnD",
+ "outputId": "a59a4610-0e25-434a-a6fb-d33394293e57"
+ },
+ "outputs": [],
+ "source": [
+ "# to save time, we'll take a slice of the dataset\n",
+ "test_dataset = load_dataset(ELECTRONICS_DATASET, split=\"test[:20%]\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "r-2LBdNVNYnD",
+ "outputId": "6c3cb81f-95de-4530-d501-f8b43f031c7d"
+ },
+ "outputs": [],
+ "source": [
+ "# More models in the model hub.\n",
+ "model_name = \"openai/clip-vit-large-patch14\"\n",
+ "classifier = pipeline(\"zero-shot-image-classification\", model=model_name)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FodfnkJgNYnE"
+ },
+ "source": [
+ "First, we can zero-shot classify one image from the dataset: "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 509
+ },
+ "id": "5FMETGuDNYnE",
+ "outputId": "508eb028-acdb-428c-a7b1-8f34e2cbdb79"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {
- "id": "B-DDXAVxNYm7"
- },
- "outputs": [],
- "source": [
- "%pip install argilla \"setfit~=0.2.0\" \"datasets~=2.3.0\" transformers sentence-transformers -qqq"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'label': 'switches', 'score': 0.9631496667861938}\n"
+ ]
},
{
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "le5C4pzTNYm9"
- },
- "source": [
- "Let's import the Argilla module for reading and writing data:"
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdoAAAHaCAIAAAAohh/4AAEAAElEQVR4nOz96Y9kS3IfCpq5nyXW3Gq/a3ff7tvdZDcX9QxFUgNSEh7JGUEQnt4XfdI3fdN/MYP5FwYDzAfhARoImJH0IAHEDDBPTxKlB7XIaQ27yWb3bd697q0lK7Nyi4yIs7nNBwu3sOPnnKiMvLVkVoehEHXyxAk/vpj/zNzM3ByJCDa0oQ1taEOvmsyrrsCGNrShDW0IYAPHG9rQhjZ0RWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JWgDxxva0IY2dCVoA8cb2tCGNnQlaAPHG9rQhjZ0JSh61RXY0CWpqioAMMYgYvAVeQIAa63cdM7JV3Ect5fbKM3/vP0botW/a3ly9fPyGF9Y455R7leljUayoatCSMEs2dBVIhkdwVwiCvCXcdYYI88LFgNAURSkyDnXWohQfzDi1yFiG9TzG5ef+gHNSnx/NUbrckwHKj4L5De0odeHNnB8pUkAFBE14LZqxM4551xVVWVZVlXFujB/gsdogWPRmgNCE8ESjg1fAEC/38caLaB2hXYsaCvigYjiuP29zi0L0UVt4HhDvzy0geOrToynSxQEgLr+yzSfzxmOy7JkOK6qioiiKIKG1iz3Wwjb4RK8SNAkQqJ+c5VSrGG3XnjHSztL2tCGXjfawPE1I21ncM7leZ7neVEUeZ6DMm6A0qPlZvOiSTZKWu+zWg1eMZc6tMHxUrO2CzLPtCyraotaTQAQ2Q0gb+iXhTZwfEXJOafVYU1ExBaJPM+zLCuKoqoq/aS+Fhi9IBXlwjYSGCsYdgWL5RVlWba+msUGAzRDspTJpVlrjTHWyvMQXDAZbOfPLtv3+rSuq3Dj+tvQi6JNZMX1IDYKizmCleKyLEUFNsbwNT8jRgnT4SPruq8BV1tI8jzXdySiQ94r9dTlizlbHsiyTGN0FEVcVBzHQeFs9CD3ouF4Qxu6KrTRjq8BiVGiLMvz83O+GVgegnEUtGpeBD8PqcN2zK4/bX0OnIqBESOICZH7Ug7ULeCs4FtFDMrDwaCjPhvteEOvG23g+JVRM4hN7ouBmFXLLMvm8znbJbq02udFxrbHI3fxSVEUGnA1Ojfvd5lfQMVHB1SVpajSbPGQ55MkiSIL3g1IBEVRJkltwRdE44lhWvkbnZYKK2roS2gXb2VZNhvbfGxDG1pBGzh+SbQi1BcayiYiZllWlmWWZXwB3iJRFMULree6cCzWCa2kcyBdKzZ1ldMFx6YRTyJ9FUUR/4qNHohYVVWapqxZS92iyCK226YX9hB165noyTZ9fWcDuBt6XrSB45dEXXO+qiqxwxJRURTsmjs/Pw/20ckzL7SeXXDc5RJstZmIzbqpFHeV0wXHsU20ZUOXIwsItnKwFZtVacZ9/iqO4ziOGa+5p2sLjKXiTLrDxfd4EbTlijWHZoXtfkMbatIGjl8eyWxn+OCb7OZCROdcURRsl2C9mJU+fpKtFozdL7SSvA2ktfKt9yWyIliby+6V4LOrnK52RSaGuodQx3toWSWuRfAuRIZj/gmouGkNsmkv5vvW2ot0rRYnui1d4mRDG7o4beD4FVNVVYzCbBrWKBYo1HyTDRcvkDpceV18IjF2ARyLmIG6dtylbHbdj0xMfsMhqe0wLAYk2Fmet9ay3OJrVpmJKI5jrWVr6xBTsLGl3+/Lt6xTL3YMdkD2uvNoY+LYUJM2cPxqiHVhdtMxIgsWCy4IDIHX7Ng8+mJr1gHHXUYGbdgNPlthel3tHslI+do9yOjMWi2ovYtRFPG1RNGRz+mhUViQF9FINJ526EkT1JMIAINhHMgYkZ1N2QPdML2B4w01aQPHL4kEpyRwmLcyT6fTwM0VLMADR9aLnsbUEcj1TFgJ4LiJxZerOcNxM3GdtbYsSzZHEJGowwcHB+LKYzVZgFtit9lYxGC9vb2nMVobmlvrU5QT0aZR7ZTp9XrBfX6+S4xtbMobatJmG8hLIkTURgnRxRhNgrUzYwfUg+H0/atDgbFCw1ArFq8r/q0qsFJ0fn4ebEeMoiiOYw68494+OzubTqf8zNOnT9kiz78CvzOwLJc9nyRJr9fr9XpJkiRJwuiceIrj2Bizd2OIftON7DAEgKdPn2JbTo/hcPgVe3hDvzx0/bVjrn5LOjGeqD4MCxwAIKCs9UWFYSg0xrDqJJpRURSz2awoiNUo8aT1er1eL12t6klwKxGUZVWW5Ww2k7Q+XL6YOFup86s1ba9I69mCCdvf+7y0cgQQkwunMWK4RMQ4jufzeZqmvHTY2to6OzuL4lS2wPAAcQAyACRJMhgMBoMB527mBx49ehxFUVmWp6enp6enjL8MwZxrlKE8z3Mu5/z8nIGYvaaj0Wg0GgHAnTt34jju9XrD4TCKojzP5/P5fD7P8zxJEiIqisJam6Zpr9fr9/vj8Xg+n9++ffv4+Hg2m926dWs2m/EbOUgjjuOyLJ88eYKIb7/9NoN7FBlrwTkoSzIGo6iWV5qjpLEjJVMzwWkrEdT3B/k/+b7oAdZ0uCI7ouNfHb3Oq4rXFo4lvF8tCqmsSqBFIJe4gMREm+c5TyEGTY+eNbWOrZPGmLIsrbVae+L7eg1alk4Ch9k0LGEVbP3UQax6sQ9tHvxFBdbc9AxuPeMDrJnufV3bKPkms7lADLtseeDSGMWMMXmeT2fZbDYry5K7Oo7jfr8/GAyOj4/Bq8y5p7Isi6IEP6a8HMnznLs9yzJ+BT/M46uj4hheebu2c47heDAYjMfj7e3tvb09hubRaMTYmuf52dnZZDLJ8/zk5IS/StN0a2sriqIsy3Z3dzm7CPhY8vl8bozp9/s8XtIJXJM4jtM0MaaWcckYkLjpZygB3tAF3fBauZqLFQAQEADKalFJRDTKX3lFMPiXhH6p4BiKsgCKZCEJADxjvSK8MCaIIdJaC2A0jIqTLXDdiJGXISNJEtYBJa0E2xl0Ngn+s+nx56pqzf2rwXH7hOp0za0Zr9WlxXfmU1aRwgAg9t88zyeTSRzHbIUgosPDQ0ScZ0VZllEUjcfjJEn8zo7o7t27Ykzo9XppmvIGEABkHRYAiqJgRRgA9vf3p9Mpl8wJ8KbT6XQ6ZUtxnufOufF43Ov1jDFsqeAKF0UheMphG+PxeDAYsEY8HA757XEcZ1n24MEDNo/MZrPhcMi4zODLOjgA8LvY3CG2ZnFL5nmm+VMMIGmaQj1ShaG5K76udXgR/TIREVcirSB71KU1b+gF0PW3HXcylUFcKGIECxNBHMUAEetH8/l8NpvNZrPpdMrqkvahyZTo9QbayCDzhJe0GkD5AV4yc2kSFIFqj4Bgtw7Sojp1NYlEV29QJ7x29lvXWy7pyrsgWWtlq4XIJ4ZI59xwOOQezrLsk08+yfM8ilNjDFuQxuPxjRs3bty4MR6P49iUJWVZNp1Oz87OHj58yOPY6/V5ZEVlZqQ7OTmZz+diaxYe2NvbAwBRwLlWaZru7u4CAOMsAysPFquxDPTHx8f7+/v8iu3t7cFgwE04PDxkUP6Lv/iLe/fu3b59m9dPN27cYEBnuSI9wPzAPt6qKjWbgV/GiU0clf8QEaNoGVItMB1udamN42I0GXCX41KHXawjtmZyqdVa476hi9D1146fQeFZRM7BdDo9OTk5OzvjSchwwIArmAsAnMed4UksFXEcp2nKdgm5r703vV5PvPky82XKtRLUzRGLxWZHqp0u7bhzt9uaz5sOdet5wXFkLSeHY8MxgwhnCs3zvNfrTSaTJEnKsvzJT36SZVnaG7zxxhvvvvvu7u6utTbP89PT07Ozs4ODAzYHc99GUcT2oizLASDLMh5fxmJE5NUPt5rtEojId5IkmUwmRVFsb2+z3TlJEtb6WbEdDAZbW1t8h1Vj1qn7/T6bgI0xp6enVVXt7+/P5/Pj4+Pz8/OtrS2W9/1+/8aNGzs7O2xlHg6H/PNA+jKf9HqpcIUW511w7FypeUlwnDXxujaNAMDw3TK+3qYcaM0bY8XLpNccjj0OLjJPFkXx8OFjthGzUULwjqeTTAAiYji2ftOwoCTHQjG7C3yzI0gnamCAYBsFL7dlqgi4N+O3hDrvd8Brp9Fg3W0La1or1tWSImslERIRMfw9ffqUnWZZlk0mE45GePToUVEU9954CxHn8/nR0dFkMgGP4OPxOE1TNhqwyslp+IuiZMPu8fHx2dkZj6nkJpUoQ7YM8H2Ox3DOsVEYvJ3KGDMcDvmYlX6/XxQFDyLH2LErj6UmIrJTsSzLwWCQpumf/dmf/fSnP719+/Z8Pj87O3v77bfffPPN+Xy+s7PzxhtvsFaud6mgD9ezdgG1pPZwCjMEUhwRiSo9lIH22hT8/UGqgXuJ7B3Au4Hjl0mvORwzOedms9np6el0Oj04OCAfpirfEtFgMGhVV8vSyTYNtjmyy8ioPL+M0YzI7AgKMpCJZVBAWRCc2mzQrDA2bwJ0enM6tVdo38XXGTCH7Tkr1hYbHfcNIiunbFi31p6cnHz66aeI+Ku/+qvc6vF4zNaG8/PzR4+fsMGXbcTsWGPsk+HQe/Dm84xbJ05UWf2A3z9SVRUPBPsMRDawEEVfQ+dcr9c7Ozuz1rKdYTAYcOwHt4UVfIZy5it+9XA4/OEPf/jv/t2/+7Vf+7Uf/OAHx8fHZVlyCN3Ozs54PEbEGzdusNFZRt/XYbmRRIttbezSF6xqaEWbPyVeRYMvAFSu0Fgs5jLmW+OT5xlj2KFX1rXydp7Z0HOi6287vgBNp/PDw6eHh4fn5+eiE1lreeU4GAySJBmNRhwfwT+pKrZCICKUpWOPE2+i42hW9v+w6VkslWyR1DqvaNPsaBI/vkB2MGFEWxG1S5eG2G3x7TJKVOuJ2+clnrvgWDY385+MepPJ5N69e3fu3OHF/mQyOT09HY/H7ME7OTkpy7Lf749GI+MjryeTCb8iTVNWJHlYx+Mx+2Z5ZBmVOM4BlEOV3zubzW7evAkAbDoQrVlM/xz3FsfxeDzmCIp+v68DQsD3/Lvvvnt4eGiMefjwYVmWN27c+MY3vvHNb37zu9/97nw+/8//+T9/+OGH3/rWt8qyfPDgwVtvvcXVNj7hkUjuosj16Dd7MpCjTasuU5a1GzesNUTgCBDBVQ6REB0AzKZZKx/20hSW5jgO2nt2dN2GLkevDxyXpYsiw3G+cWydA2PAOXjy5MmTJ0/Yq+6cOz4+vnHjxr1799j30uv1rEUd7AkARGCMxJ9BFBlrEyIwBkaj4e7uLn/LPylLx74j1sUmkwlHGZ+fn7M99Pz8fD6fcw4E8LsPoigS+0bT1gEAaZqyJZrdR4zgVVWlvZ5WhVC55sk7D0GZsyObyPNaIWIFH9R2bV+38KjT57t+EvWTfB4JNhe8+eab7Mf78MMPb968efPmzdlslqbp7m48Go04GoGIuJfYUKCVQfTGaI4c52/5mpswHo+bldna2uq0uXsbuvxwe3sb2oxC/ORsNuv1elmWbW9vTyaTfr//gx/8oCiKf//v//1bb72VZdmPfvSj7373u5988snOzs75+fnR0dEbb7wxGo24N6Io4qh2tpZo0gE5K+oZoPagI22/CmquBVaKgOQOkWbOptNWWG/Rps0q7Tk4svaZYftdD1BjY6owxqoSrwm9JsYK55bR8gzEVUVFUZyeTPb39w8ODngvchRFX//Gu9vb29vb26IIOwdFUaRpV55fQATn2OAgWuriLV3Eb2frM4P1kydPJFuQbCiQ7JrozRdi9GAVqd/v83qWqSxLu4huNqJZiztRo638pCrnwr7kXfmMgFgP3eMVgLEpdER6NKcldKc06tKOUa2+uZCzs7P9/X0WZtPpNE1TDkXg5MWAluugm7aifGsjXXMBsq7p2sX/nauNDviWnSbz+ZwXTOwons/njx8/Zky/cePGv/23//add975wQ9+8OWXX37729/+lV/5FZYuEmthu4z9HbSuAaFrfdXVn65u9BCS1Ybxxx567dsKTDMnd/tHVtHqE8dlWPH1MqG8DnBclq4pmbN59ejRo/v37x8eHmZZlqbpnTt379y5c+/e3TiOeG7qmSVh9uFazEO8c46nCqvJF6eqojzPZU0t0c0MQGxYzLJsNpvxkR8cKtfr9Xhdz+pkmqbsPAREo1IriE4iE0PsIUx8ErOeSKAslaB2IrB7Kkn7Qf3FFqn/lIumNhc81vwCVNwIq7EcjfDZZ5/Fcfzuu+8mSSI2nyTts/wAdeApa5StxYt2H4iTdSFhXTiWrUPcjTonCQu/jz76iLXg8Xj8k5/85N69e7/6q7/63e9+l4iyLBPPsIj8gDrF25rt6hqXTjguFzb3gCQWO9CasywTXNZKQ7/fl4dbsk4vq7fciwjQmIwQ+hZFx389QPl1gGOx4hVFFccWAM4n2aeffvrw4cOjo6Moire2tra3t+/evXvnzp0oWuxxcg6cY2hAY6Asl1jchOP665acRATOLVQwUTPFCICIfKRbqyrNGjc/UJaOFSvWmsuyZEs3AzQ7uID9M0oN0S7EhWlC2UB4gsVRBPVQJ6Y4jp3KgibPl23aECgEdyojPhGxiaBJndPeZ9xnM67xKdmm0+nx8XGapmyaYFNGURQ2Spw//FRP7y5mQFwGIAoWX2K6XgKO5b26i6IoevjwIcfwTafTzz//HBG3t7dHo9Ebb7xx69YtjvTggUNE59bbpI5dGT87ab357splTpKmLA++atZTC29UqzdRI1ic67XdirhpInDUnrj19aDXAY6ZWKgaA7NZ9uknn3/44Yenp6fG2DfeeOPNN98cjUaDwWgwSPlwI/Tp3ufzOS8zZddTIO3REMeWai2SmcAYY63BUFzXcg40uUX6W0zP6DPqwlI7IGsxz0sxd3Bk7mw2e3p0FGwLZr1MbBqMxTy9jTExfyobCAAQkcSliimAHyiU1qMvxLkUYHRXgF0XX3HcMXgnntRBFhBZlnF9uGJoIlRuT/Kb17uMDwJPgRbfNW87t8+sGcHS9e3R0dGNGzc4UI/f9ejRo+Pj4z/4gz8w3i0podOIWFVrGn/WhOMu7buLRDvWdQjGXddNxHNQYWE8vWQRJUZh8cL4IEYPZQ8BxEW+abGn6wm7VruuJr0mrjzWsxChKKrPP/tCsHhnZ+fWrVt7ezfTNK2qajKZnZ4e8XwWEwG7idiWCg3EiWKTJIn16SkYx40xHO4ax7HyYxhjsCjKOI6CmLSqIlGTBR34WwnLr3/FEfsRG7i3tkb8TFk6Y41zxCqV2DeKotjf3+ebbLuUFBDj4dYC1xDZNs3QrM0dsNgBXEZRRLiAp0Ab6oLjbpjovK+/4rrxzgveZ2yt7fV63AreFCdTV0pYoeq2fiXz/8WRuCVlPxHff/PNN09OTrIs403ebF9O03R7e5vDcjg+nQPyOqMPnx+tK2a6ulqXo2G6qwlijYG24SAiDmcM7EtWEU+w4SjVyjUAcCx5l9HsetFroh27RRoa9+mnn/7igw8PDg6MMXt7N954441bt+70+33O9nJ+fl4Ui+W/8wkbedR5q5gUKNeOShHUvK2DcY194lp6s3Y5HA4lDQ2os4K6dkOB14iZNPMzQJPPL8MlVMppqX9uzCJjEU9yMXqcnZzzIkAEDwOiBN5FnpjjCStUHnOx1Qq7a4xegYxdRgzj4/xAWS1Y1eIRYbHH5tcoimyUsOx06ky8FdPeNHIsiF7W+vzzQkCJ5ahU6lS+P5lMxuOxtTbLsuPj4+Fw+M4775ydnTnn5vO5zkhVVdW6Sl6zvatpRcRCe/mArc9UjdNe+JrT3UEjFFofv6v12eC9Mu/E+COKMI9UkhpOI9Xv9xfelNeIXhM4Znr69Pi//bf/9tmn95MkuXv33u7u7ng8riqSuOAir+bZmaQ91ObL+XzOhQQdEsWLDdMSQcW4rJ/hO4LCvFsXVDpzMQUI9gmKtZo7nCNjUHyGRcHpKC3A0rUoOi9PBGpkYlyo2w6JgCOmOZ737OyMt4oxTEuSIy5wMOoZledX6s9yqwnTXUaDThefz5DJSg2/QnRktlGILXU+n9so0S0Ftem8vXw1evqZdW3HXfNihdFDAEivvnmfnrX20aNHt2/fjuN4Op1mWRZFUZqmohSzuawsyy7bbld92HW5Vsu66t9efsP4I39ig8BH2nQtnprwrZc++rPrxHRHuXQy43Kv1+O4/ot2wBWmaw/HjnIAMBhNp/mPf/KXP/nxT62N3/36e7fu3HHOnZ2fn5ycFGVuIutcOZvN+m652w3U7Op23Swe1pPN+CN/wG/0kFABh7UgB4Et1kZFDxVcjuPE1IM3rUVrO0MvK1dI/bs2tupUtlbZFvVYc/7l+Xx+7unk5OT8/JxcLOlApeHOua2tLfDhvYLRiJj0l80EtcCkxlaXRf1LlIclZAJViiXuHPTb57j+AhZ6JkvCaP1nlLSLBxEbwYqYj0ZtwkEXPxBYRhyWqRLyEXCUfLoqby2nizpzRHVQ176gLrHR+d41caCrfOwI1Fst3rrgWA/3YrWBhteLAUvs7OwkSZKmiTGL9eJiwK+VSfnawzFACWBOTye/+MWHH/zio+Oj0+29G2+/9W6a9uZ5NplMJrPzqqqMAUQkdGm5gOPgUxaYGMbVslW6BscaoFEt2ImopFq2IFmUOa8VCngxLidJKoYOsR6wbVo0LIE7RABcevCZpP6L5jRCgYIn618uW8RmzfNJwVuHOXiWPzlxmi5ETD1bu0O2frJMkt7o9XrSn9I0RHTVokVSZzH46P0O0v+RWRgBgvscqOtUbqbFz7F2iooeWd1keYAPf8KGNtcJZ35XjqyuQBltqEFxp8uxQ5RecziGNV2FQQXks1XLRkQDtTMkQc1Z1pH7/X6axjwjiVqOmiWf2/py9XyhdO1deQQGwOw/Ofz5B3+9v/9kPNrmBDTHx0ezbM46IIBz6NAnlGhiMRO2URQtU7po/JXoOj29pXBquGX4ee0+NotkNBVDc6BT8648Rmf2ubHOmKRLoy1rqQgMKN09xP9RbcY5f/gTLcIqrLUG0fZu1HZzleVigwNnpxQz9GQymUwm8/n86OSpYKW2QXPeOw6XZtMNvytJetwirSATUU/tNhT91BiDdhldp4dJEqKKzFv8aZYmez2+weIXvT4e+UBAqMNxV8SIlhli0+zCcVxl4+7SvttpXXi9LtTUiwNRimrlRESIhhqSg4g4pQzvseLMeQLK0BDJV5auPRwDmLOz8/sPHj49PkEbp4Ohc+7o6IiTKzoqDSIuDAiIVMqaOvhsTg8/28NAV9HFxL0rg22tBWhnL8F3LpbncFVVVeXQH1O/aI8vli+04oyIabqwSjPYcWIaxkGoSRTw0R1LmNBsLK7FOm8bib/m/ogiiKIYIB4MWNtdFOsccHKl6fyc0zNxKg+J9GBXFSuSXFvG5ThO+VoUagZTztRj/bHQonX2kqVNEJUrT+A1gDzqzmUK9Wkpd1qxoHPqGpTwa1DC2C6zTCxfh4hUvZqzDdfVdp9b+Wu+VsZO+j+YNY1xaX+1TC4ON5rP5+PxeDDos2kdvc0NfPT6erV8WXTt4bgk+OiTzz/5+NM8LznaYTKZsGqTJIkxQGWJFtI4NcZUVcGe6CYcd1Egq+VC9NzgYUe16aehXJQ4bTBFXLqYdRSq/JC3OUk949ho9GcF2VrL2S20Ns3I1VN7rM0ynhqyrIiiSCXJAKhHgPgIaG5FbeeLNM0Ys7e3Q7Tzxhv30B8MyIErPCvYecj7WVi5Pjk5EwXT+F12xpjxeCzHL+mFwrRcJkHVTdD5fKU+AEC4PDK1VdyGQ+baxWcXRZbtKmIdWgxokkQKUBaOPQAwsOZZhR3vXc2lFy+/i55X+Z3+jGdFtrQOQduI1A4phxpSL9iDHdcAkGXZeKsvfg7sXqxcEbr2cDydlZ9/8eX+wWEvHURxKikUDLkIAZFyKqBEhDiyiLA8ByxAZE7vDcotgwsHUcd2Bh/IJesp4iyOJsQCfaGlPf/Jpz8EuhvDEdHy1AzRwjiQSGLgRSpYtTFPDNDGmGF/gPW9IaxlR1GUpohJpOMxEAyzqwCxMbXYO0FnRN5QYAu/jYXvx7GNokGv1xuPxyKwOIMHJ/HY3z/Q8dHsRcyy7NGjR2LG0UJl0Ovrm9IEDkzUElELSN2fARAEkxmXI1Pjh65cHPo8EVAo31wJLUa5wxi8LhxfF3om7LY+H6jGUPfu1sexNl5CcRzLcQQCyvP53FHOiRtlxenl6FWkaw/Hn93/8uGDx64CNhlzjD2Ai01iI7RoTBoDAJIDAmsW0juYe1xUYKiCxciTWvjzZON/pJY/y9VQZCNdJs9KACBwyMf4EgE5IAJOp0OAALwAJrbnAoAhdtEsGNT/QwBXeqcWIAJS5W2XhA4cABRY6Po/xacajq3atML5dvkrsfAOhwPGNFwkTqrFeMgFyS5zw44XJ/AURUYHWRMBK7Osz+7ubvPNqgI5Wmk2mx0cHEj+YgZuzvt+dPiU516g+HP0mE63xNelWySWNKaW2kY89VCXkVQtNwE3WaJJFS3EtiNnkKNyCdAVZYYNAkDXHq/1wum6a8dazmmYFkgOChd9SPiQ1Hm1zrler8fbuNBbGtdq78uhaw/HH330yeODwzSKe73B+flZWRRRZFyZ97e2DEAUm77tE1HlCqictZbqLheZmXp49PCzAw19MgrRSfUJTIxoPMb8s4DJwKf3BrE7eu2YDAqUk8+SLhK+qbjxScmgWLDJ6JpZxVaONWMFSswvwxbDpbXWmNoBVGI0iPxBGJziQwH08jgnXZGyXByBLMJMNO6qAucIEYfDdDhMb93aBQCAbwNAnjsGZdkwOTufiv+QbdN8DB2fzcFShKvH14TO+Nx4zZ2H0hVyHdtwt1ig9gZkIiOlsTdVa1vBoBBRhK9DPOzFaV3tWH7VupSpATE/0FECn3srRwYvnRYxsoDP83w0GnGe26uJxQDXKtCNq8rJDdh/+vHHH//rf/u/FEWRpsloNIosArg4jtPY9pIE0BkgAEBDdiG0nYPwDDFdvgy/TFrjA5sYmwRJOxebCBqmjY9Q1qczaFgs8nYPfhcZY/U2FqlbVySAbE6VfRy8/8KoAA8hRAQgrUfLphX0pmre7SJqadLjTauygFi8l6vDsMt2HUmZyFZm8ptcJDO1Bu5lUW7xfFUBYzFvozg8PJzNZrzTUnayAACYxQjy6SHs50TEwWDAnGOtnc/nSZJsbW0hIrjliOiVMm9jkdgJYYwKcomM5pvod2xSPQCLy8ymtXT74I0qEjGt+QEACp+zQstgaMOpxUW1PE8P6iwdCAn3UvIClz4FUqDABpUBqKnDTeqEb1c7QVj8EAzE0Ow3KKIo4meGw+He3p4xhmW5nulSsVeL1NdMO5a+Y230+Ph4EXXkAIkVFptExloD6BAJCNDo8V6xU7nFpU5EAA7AKanM19glpz2jsMYNAI79YOxCJCLGRiL/xjVTwBCFtm+t7rW2qzkx5HkJ3VUNXx5RofFa39F7C6MkFo+i6KTWnxmhX635XNeIr1d/iwhRBKNRbzTqAewBAMB7zsF0ujBAM0zneX46OeGAp6IoptMpH4HIu8wR8ebNmzs7OxxYned5HMdJlAbdImhu/XYVvrMIbokiXDgVKuk9ItK7Orkc/qrX68lsl3QWelch+RjbxW9tS0Y66N50rgdaX1Dd8wl+/b66kOdIAnZBrVo/L06BNoP1k6tan2dpzSh8enrK57nIWDel1yuk6wTHwlUAEEXR2dnZgwcPkMCisdZEBiLkbJlokBZpttHxRHPS3R1CV2aRTB7/2T5IXYOHBOItdFUtUh2ATce+5AW/rtsNDoEQAYEMorE+j3unpGFOZSMNIXJL5T7P+eWsMD6VWrBQsOpka61Q23hxQqDaZxhba8nn3mRLCD+QprFkUwIFtQChdkyNPd+sbqOP8agqQKwBNBGUJRVVzsYNzigth+k9fvyYj8XL8/z4+Pj4+JinaBr3uAOTJOEj+BiIz87OOI5QDFMLILBLnyGjAaJBMDLoBEREjNVEVGJte3egTUMDlbQA0Ajbqi5AA2S1lUzmi5Y0LxORoSEk4MLa8Qr1Qkx50sYVSIreK2utZX9yVVV8sEvzh68cka8THDOJAffo6Ojhw4epjci4XhRHxlqDxlsLABJEYo8XASDaizit0a8r9b6DtUh4RaJusR4AIC/yU3S98okqL1JEOK3iIcerfSBAXpaTQXC1NOeoPoE3Ffp3ES0jQCppCOubi2A15R4ElQUf/WlJ4LeHxHHc6/UYtwFAtrrwk26RPASbyjITL7LZakG0/FMQnLez9/tpv59ubY28+3NBVVUdHR2xe3A6nU4mE56ZJ0enYpg+OTkR+xKLEBEkYpsGQ6a+qV0QQRYTepTZLhTcxHrkhjCD9LOUJvcl8gdqjmIUl2/wqXULPZorWGUtWgGjTT0dGrj8TNW4Cxll+5L2rzyzngLfkseRg2Kf0ciXTtcSjllVOTw8PDk56cc3AG1kMbJoDaI11hqiykdBMDdbAADOfdUdlq+HVuvIrfSStQz93mbdLlgZaaBRkfDBPJH081CfwAwr2iXN3+bzeV1bRDGyM8jyhlQ2cfR6vaoqeanIcCzA3ev1ZLeL0aduExiDYvBEXAAxa8dyU+5LR3jlemE4imN7794dqmcCybICHIoNmrVpNnScnp5yAjbOvyzbcHqDPiozOlvSrbWcnU6HEnIfJlGsO5OUpaL5CQAOFnxrGr5cEY21oazr3cGaRr/65diOwSNylxYc3L8EJqLXi3VLu6YAeS8LP8Z768/OzngN9EwT0Eumq1Wbi1NVVYeHh1VVkamMgQghMmCiRZSV0wZTYwkAwRKwrtG5aTXgDFyZV7dr+K1X8JoKEdR50TPTutPDeXbnxG+LU407H19AEiEu/vG139RPALW4ZmMi9PY4YXdpi1MHcywWAeUyblq2sbBNwPoE8zwfWBvNsrlsYOO4DiJi3dM2cndYa7dGY+lPVOpnkthgIpM/1TBIUgoAiJBlea+XiCrNF2kaI0Gajre3x/yYc1CWFVsb2fTM4RysPmdZNplOy7JYnCvo8/qz8UFs6HrD4Xw6C5RoWUBAGxiZyOei8nuFliNZR+dFq0HGcfkYKJbznbPEx+dCnUWFg0JaVEADndclyX8NdaG1Ao4jb+7nbpeULHxiw5XaoXfN4Jh9MsaY+Xx+cnISRZEBsuh3QKCJLaIFqgSLGY2sQ16tdMpisdnpwV4B3yv4SfvW5eGAC9VUWXObEDo0Bha7dMkREFHlqi7JQQSIhq3WAItrRAJwaoqKNZniOBH+1gF5cramwITcFyzWrioO7JNC+NSSLMusDW2jrL/M53OjcneIQWA6OZce067CLMskzMMr1MZazjVhpTPkQudE1bZp1rLFTm0tW1EsEQwGvVu39vh+USzStM6yLMsycR4yUhdFcXh4CPUTC1kwD3o9LWbEBWob6fy5Gy1FAZ6KIIc2rdl5NgyA27kwlRV8BRC8BFGdVoPmxcnUfRvPLFC6hR8W++FsNgMAHpevWKXnSNcGjrkTnU/FVFXVbDaL4ziCyFqMjDWAAI7IkKOqqiIL4JN1Lfx4K5NlBdJbZgKBA3QAfrfG4hoJOlLM0DLQDdAHulFroJvB9QPddIcsIp19EFtXw3Qbwc/zINBKvgVCzcF6ZS0Pkz8hyTkX1w+1kt6THxp/WD2/Mc8zEVfyIkTMskyMznyHQfbMn32HajOLqKUynYwk94iNuBNBaaOwsG+g7DMsispaK4fYliUvDoy1S5iWLoxjjOMBwEBmP1urs6zSoDybzXif4eKk8LI6ODjQhmYBa8nYp000iIh5GIAoQLbA33pMOgd+6QGCus9QvmL/6vNC5G7tGIPHgiexYWpY971U36eqebhJAt/M8+SDZDin9lXbM32d4o7LsoyiiBXkjz766D/8h/9QFIUB7Pf7g8FAH8ChcQHqnFoUhZ72On1E60t1RIH+bJ6GwM+vtgw2i+L4R80Wks8hUIj4eT5i2fqD48AHYHGgVbO9XexeVVXrw10MunqT68Xvd5XTVU+C2NQXpDy+rO1aFQLM1wRGMsYt1kwSk7fc1GdwEX0IZQnWrjoanLcmSkRHUVa+Pgs37DKXIwLA4kyWPM85N+nBwRFr05wAj2PGuX+kSuIw5GWfMabX6w2HQz5elvG31+uJ5Ue60SoK+MFaywctalwmv3LXXcr8Fi1O6qucSlVora2ycFuhnjjN4aal3XiJm7D+zguuDLfL+XRdemowA7MMFtBvsrSjWqJOXX8WiuPxmI9Auwp0VepxEdILrtlshoi9Xq/ML7oLVU9m4UVRH7q2UbQWcgkSz3jzs1V96CpHQib1zwPZcJEKB2ovdEwzXasLtnT18+veB2hvFOeIEckkltx5VughFntuoIcKsR6tzjxERl6xZmD9VOMoWtqsuWs5RK8oyiiKjIEoMlHUHw77u7vbZeneeefrfNQsa9CcPJoD79i8w8n+OcWSc06yLugNh3Ecc/p/a60c2MjcK8Yi8EsKFlRs3dbuO+koUKMsz7jF8WalLLn4h0ih9q2ZVtBW39csty7bPJN0Y4UZlsJA+TMXD2DtvlabyKfl5IGjjgMfXiZdJzgWtmAlkW+auguuOW8DdBOmCXZPdb1Uj3Tz5rqVDxoCSk3WrVtRHznKSFemVSVfXdWgDqtlADy/ebWulkTUPqasZspKiEGkKAoCIyZsgSfeIxtYXRmaZfeKbETk+6K9SqQEv91GoPeIAywgm6c0qPhoIiDCJEGAqN+PtrcH3JyyBOfce++9x+AbHArOmfDYNs0eJx7c/f19gWnWpjmX9GAwCIw88/kcvU3f+KhNibfj1ZjGMrlgaJbtLQvd1hvZTP2MO+4iUKdScTmBOBeYfr4UcG/QIs3JHE6lp5Ve3XIgDe9hAgj9wC+frhMcy1oM/JzMsqyXpKI1d6mHGmv0BV14T6SA5qW1Y9msLEXpaoBfvTbr31oZ58/xRH8KRvNXwYsC0h6Mi0yYdRu+rhbcKTagfTQDkaPs9aE5iIW3ZJSWcQ/sucaERlvjLdQ6L7OxFGjZoltJFAd4jFYYIc2HOAYAk6bDtk4AAMhzx0DMoMxJcDgOj80dnL9Uq73RMp30oqqcdYTjPbQCq0O7dEdpa4AYNKy1ebFIyUTeeSPoDF4f1z7e5lJv9fheggIGIHW6RzBPicgouJB+kB/y0oStoDyIr5auExxrnhBTqbaadYnir8IKrbh56dJa9dYmB694hVb9QPVAly1vrZrA+jB6kfK/CkkCzKC7GAUEGoQBirIEteJhLNbiStbX1jvyhG00MEnJtp5JrnKF6NTylfVxx2wIZoNGoEGHNuhiCdxQB/E0NWk63NkZAtzydYb5vGBbcJ7n5+fnp6enk8mkLMvT01O2hHDCAONdmvP5XCcYkV3sYuzSskf+ZO6SLSrWWqBSOE3Pr7wwspJAg0CEwIjcrt+syw/PXKs11eEmFssn1v2H4LcvkHdK+5tr1fH50zWDY77ggFD2bzA/QT0iAuqgpgdGb1fnb0Uv6Hpvc3QvQcLiAXPLAeZyn1aaDrSZWHA54E7d/K5pIO0Neqnr+a6A+eelNXc+v4AqYjWIiMDHuMByzwfH6iGozYfWLvZGOnIEFcexICIa5IM8AB0XyolPZRobMoaWKZnQZ+9kmMuz0tQP0+Jr8jut+WBjAbj+IMVlBMWy9R0HbS/Xy9yv/Lwx0O/HALH/+Q2ihVVkNpvzNhY+jYX3sDjnjo6O2BjC55ORN+zoaovdg20gxic2kopGkT++vBFgN5lMrE8EKPt6jDFVWTOjCXs8L/EspWmGDwwmWDdN6Dkln+gjjtl8X1UVZ5V5tXRt4Fh3JedXhHrUgRA21jJQl5mBSgUrMUV/9by0Y/2p2Ug/s+ItqGyCEvbU1ZzV00B6VXfOJVrUWvJaz3e+d4FKoXySOgfiDQnIOecInLdELa7RMCIbdKgSnMphWgTkiG9xWLZ/P1WOXFkVmCNi5cI892K/JiLeXsj6MivR59PaoS3SEDZeKc0Um25DcS5VFfDh4hJ+RwQsH8fj3njcu317l3/iHBSFK8uSjRu8HXwymbDdoyiK8/NzzXJSAdadGaT6/T7/mSRJARWqw2e1VYelHVuBxPKeuUJPE3nXuvvfVvP/Yuzqhm/NQsoIuUzBGOg66ANLvL0i3Fj08unawDGTLAlZsmVZlo6Sroel3zX/BZudmnufWumrQDBTV8qYdUnPf9Z6ZIka8Jk831qOwDd54mv7gqPiuxreZcRv1r+JwnLB/SCeKL7P2mtRFAIl3F6ewAwT8hM9w2XaywPOuSjuE6FbpP0kAJJA5rIs87zM85JhlyG4cpk2dzRxXMwdUj0511XQWTRlGRznlolJyWcr5X9pauI4GQ6X86KqYD7POT56Mpmw6ZlhmtPglWV5fHwsphvJZudjUlryR+/s7AjfSF8baxFLPcpNbeP5UsC9Ta7gw9KaeobcYYlSlmVVxRwB+Qrp2sAxz6LIH9BZVRVzRlVVfA6IBAAlScJudHHvMFRpXVKPItNSXfLEE4YD1AQsZBbJBNZqOLVp5fynGAcCBVlYBxq8y/c1+Drn2DMuFmQmXnBJUVIT9AmOnSQkQ9RFib6mMU7/XP6MouVZcPqTIz00/InCJe8V63bVffQGec9+UE+CmlVXOkdfo194cgnyrTZA9Xo9PS4isfShXDo+oZZDWb1U7hvvAOTOF9Zi7xAA8MYWNBVi6CHkt/BGRwBI01Q6kxNyGpWliGvV6/W8LmIBoCw5FBeIwFrQljaGZj6CVu70eslgkMidogDOpMMYXRTFo0ePzs/P+WQWcXCVZelcxaYMNkMnSTIajQaDAcDpcDj0CnVsTFSWrizzwWDAi1fJo9bUPzSAxh1WG57dMqAiPoPJKxNcG5c0X8WJZVuEhDPK2jqOY47y5oCWfr+/0Y4vQzK0iCiHLQXTbPlAQyoGyAgqCDcoX6OS/mqFobmLuraNrCaBKl1b/VuspxnU/cCfAoigpoFMkhXUVCi02JD3yvJCLPh6tqBPo9Usrfk63TNLiWjbf0XUPgQqU12Nqqq2HQCRrc9cTiiVqZbxrl4+GgkFA3AAi2jlsiwBanYzIiQyZZ6LXAxkFe/yAG+4YAEpDwsi85/D4dD6/Mug/I1s1NZqBxFZa7RtgHPgsaGD9es4hjhOR6MUYIcInINvfetbZ2dnx8fH8/mcbR1HR0cnJyfHx8dFUcznWZ4XXJ/T07M4jnd3d/lwjV6vNxpZOR00yzK22Djn+BgOUEfS6LGGOkavIN2rwTySDrF+M0jAgTZaLpH1AqipClRVpQ8VeyV0zeBYz1X+06j07aQMQ1AfRa27iRYmE4M6tFqZG8H9oPCLkLaK6M+LL9KhgeMaiwPJQUrZRx+N5PzeAa2j6dfxlJbu1Qir3xK0S+AD6qt+hg9Z5jO+dM1AKUe/0RhTdBwh2mVU6Vpt8lvRf8pDqJvv+w4Rbce4VAQaE6Ge8lH3vzYHgd8GKWzAscb8J4cJkd9YCPWM0jxSp6ennACPl0HseeO1oPxEzoKz6jCtALmosdmBMdpak6bbnAjYGMjz8vT09Ozs7LP7Xz59+vTg4OD8/DwvC8TSmMLafJ6X4+l8OByORiNC64AtGzFHYkjUCjQmDtT5v0uz0ROtCd9NhtTbuDRMs5Iuz+je0PzGFuQ07XCwviy6TnCseV3DKyhLaKAtNn8eRZGMkGwMXfFSDVvy3mbhz52Myqkmb5TKNNsoAkYelj+FoY06RlfLGN0W1mSbcKxT8EAdl6VK+k/BERkg9KFUre1l/MK6sQifX8ItjQ5aewpkdtCWJlWuQEQbLXcD8qLe2mVwm3OOgzUA2utPanlBKmKX1AZL6UBRGubzOcsz9AZx8Do1+NwgfJOlr/Un0kpMnvxctG9jFvKLPzlfByL0+1Gvt3fz5t7Wzs39/f379+8fHh6yM9CXEJWlOz+f5XlZFNV4nI3H49Fo1I8jIpJUBGxj4V4KmFna1dXVzQGCelou6Tf9sCYAKKtlvmkt5lGdfsndXhSFPhvhldB1gmMmPZeko1HlrNEDrEEH6osm+RQtkp+RefJ8AVfYSKPkiudbMQ46eBTqjCgqGPqIC7GoorKha0jS2Kpf8cxOkEVi8FJ5kbxXsOaFUtf0fuYqJFDiusrBZfo3ZpLFn4Kc/BQR4eI07vaAQrHFgxpT8jG/AaZwl/LOOtYh2DsSxzEnMpUywWvZ5HcVaruHvi+5QCWhR9AtxqAxMBwMb+zdAjKj4RafojKdTsuyHA7GC3t9BdPzeVVSWbiycHanj17REd+gxPYGPoYV4xhoV5pX9UhJH0rbSXkyiMhGKMsyWR3KYlFKdn57+ot2Za+m6wfH0Dao0q3CcFA3RMqfLKtB2ZLEWBHITHk+UCQvh9RNGcD3uxQxzYhNZGwiZjB7QSGjJBXUeG28q1NXD+qbsLVO0ZwDenrIA/Je57fbinbMd7rYHdusTERUdRwX0FUOUTvit9nuA8NXMMPbxyWKYm1+Qb8wp7p1SN4iwyusiF5pYNZCddJKAAe6H/TzACCbDJ3PQaH5mR1WesSN3x4iRiHWmmU/S6+3OKQqiiLe0cdtyTIEwH5/wMe/FEVJBGVZZlne7/f7/f5wOEySpCzL6XQ2m80NjQeDwXA41AzGrjauqojkLs6XbzUzy9Bo44PmeS3pA1YMJp0USD6OiDut1Xn7kulawjE0PD+a17WnDpUs1bNFjzHUzxPTv0VEdpo373/1+q8uRFde7kDd/tV8HhtwTN52rNurNyQEHSIxG+ANIPynfl7DlgizADuCGAlRTLraK1q8NIT/7PKudLnyunp1xUzDNutkZ1UXqaIXEt2YhXutqhx3IQDx+bbMocbErRUTvsLu1VhTyKEP+SCfJEBMFlQ3fbAPTZ5kkCWfegnVhkNxGKI3OovKDACV61VVVVVkbdzrDUajAtGWZXl4eIiYR1GSJL29vT0iOjs7423cwm9crMTMMeSB8qOsngLBnNW9oTUV8jlMsEH8bLBAEf4kFdZJysP/CulawrHSaGooEFxAx6Je2yhB6YBN/sD6yl3eaFbu4uuq81rPtzIrKqkTVAzq+CvfslnTqFA5bZRsKqRBaajkVuunPkMP6lis30vehdhlr8D6ghS96uc6nu+Cy9VLYGjYjkWx1QNE3fHXeSPQDeq4ydfadKZHrclIQlwTW8+9IGU65zgmjK23/JgcUK9FpvQheZ0aGvjFwkD3tpgv0C8Qe71ekiSz+Uy+HQ6HnDGjKIrRaMQ7sznb3NbW1nA4JKLD/U85MMM51+/39flbOjoTngXH1KGvaNsxqIWyGCv0TQCIk8XWO/Q6irbUX+SNL5OuDRyTX4UJV8nmVF5fsx9DW/e1CUlKMMztALDMKsDfOmCX+0LHAUQEdLJ4R2Xt4mWjLMfkW+Pz1YpRG+oOBKmVXIN3vIBKZSumLmmOoJVgHCsa3AliGQh6rKoqGyGAIyBAiGLtlqzQkA8Dcm4Bl1iVTldbShNwkbnEEMyp9QJ9h2srChooZGEdrVmOILjA9+LbroNZ3RLd9Ntdh7HCqDTkLIw8aBK3FRHkpBIicq49oiONVHZd4v19BEQR4sK+gQDWgBVWWRwZvuhVqQ9A4nfaRYhA5IrCACCzBwtLjyORtaVzrigAgM2xHGkcIRoqoPJ7xsuC3+8cFFlmrTUAiISIFitXzGbZeb8/dFVFVYUAhg36ZVVVFcVxmWGmcnRkriwzezSZ93o9H0tn4wQGw7iskMicn+dlWU4mJ19+eX8y2X7rrbfefvvWnduj2Wz25MmT89ksShLKczefczC1tTbm3NJluRAVZRn7FBrcqxJEaK0FIEcE/phzAkKzcKUiIm9rBwDeawlAssMbANnGZa0lx4e7somfTzFmV7NFg1VJiBaAyrKMLM6mWbw9aOe3l0LXBo5fNAnCyh0edWuX01irNtrlwjc1pEKbOhxossFX8jx/JS5swXH2hwQJNgPNqElcf1iijKyyDTW0A6jb4IL+0ZUR0BR3TdAbYtwUxVl0N13bQC3VaL5ae+qizgA4tS1I5n/w9pdJXfXUawto+PQuTsLMgZDm6DpS60J+C6eII79iEDuGsymzuvdAWFacoyjKsoxzZczn8+l02u/3t7e3b+z12Sp9enp6dHTU6/W2t7ebPgNcnsdY2zXX1diuYRKxvVbnXFm6lnB8uYnK1DWuUqYw5QKzXItZgxrnisvNYMOFLlyjUvCp0acVWzX+6glGyhXZ1V6tn2pobn24a7Hv9cqaUQLquw11o0Rj1TNNnoR60It0ppSjPbSt9emidSMrumhdBuuuZ3s5XfUMxuvSfK73jOi6sZkCGkOjlVNJjmyMoagiIjkp0VqLuNjeNhqNEDHLsizLnj49TNMEEcejb/X78WAQ9/t9yQ83nU7H47HwD6pgfxHPWh0xHQlYAr7VzwSYcE0B+trAcVOdBB6P59TvOueDWAkQ0ZqlNtp0bQVMg365LUYMqCNv0JAVcKNzL4h9lj3mQQ+snrc6k5yWChpG603oDAgjv78O/JwxxpRqmwYpA2Vgq9GaF9SjuYMAj0BRWgFba93XiUmZRKa2Pv+iqatdejO6MNsKcdtFYtSS8ZKeaYId+bjyZrDN2eycqHJusEBnIqLSOULEtBejGTCjzufz/f39PM/jqHz//ff7/WgwiL/+9XfPzs4PDg7KsuSEn2z+lv4nWlhyqL5gQpXpEDq0h0BodcH39ULlawPHTAHAEVGXaXF1CU3qYnerTuLS2rHwkzZcipNQlxnI7WZ9mjxH3vGl0V9cYQGmrKj8M0n/kFYaK0Bp4qBwUxahgXbc2kBYE0Yv0a7uALi1XalrPb9u+V1iQEJiQflFAaArt8Nq0rzHFxJfLFFx+mE5cJLUqqssy9nsPIqiOE69LK/i2B/7ABUaAoD5fP706dPPPoPRaPTmm2/2etZa2Noa5nkex/HZ2VlRFBzpLJo7IlblIuVQy9QGgJX8E0Cw1pDk+nppydcPjlvn/FcnXqBplMGleWvBEK3asQy/1ua0eig1Dy7k56RMuvIWWTCKHqc1F1B7OlarTs3UP0wyHwI1v6sk8tsHgukaKiO+A5vZgkTrCSabaOuti+uupq0F6+BhLpBk2L06ee4MFtAK+JYxkjoE/axpxXjpcZdR0BpDsNKSngHVfN5WxymBnHNElbUxsCsYiePnwJ/NXFXF8fHxhx9+2Ov13njjNgAYAzdv7h4fR5yZiIcgTVNe6gGAq/KAP3WroYHCVF+YKr6tcXLA1deFrhMc6/lz6e5eMY31XAWPd0XevsgVpmmuyvmxroicJjjqr4Rk2wIpFUnjSKCJd0/XTuODbqy819pO23FgK2fnWOuRrKDgIPhWVsrSIvCZ0pw6wxguC4hd2v2L3m3VzYrtRomuepYlVFVRVZXf3MydgF3ldJHuXib0e4ICRQG9HUxzGnj+TyIsgZCMc4uwYYzJWltmziD24iiNbC9O0ig+Pz+fz+dZlj1+/Hg8HqdpOh6P09QYA/1+/9atW2dnZ5PJhAOf9VHfwu2amQMxH3wbXKy6udGOXxwJ2D33kiUSHurDL8aKQKviyIHAtcVPGrWppFXCQ4N7mnCMKj5UYjZs/Qxz6QfstrF2ZcAKQtOkDivignU6C2mgTlIDav5rYG2qaVrT13IlADW6lM20qx9axeqrsh131ZMzYEg/6+VR6/Nd3ROs5KQbZdUFvnv5T05OJMlC1abqxRHdbnHudQFAAAkLDANRVVVVVRBV1mIc28q5oig+/fTTKIq+//3vA5g8p34/6ve3EJHxOs9zosWRg0alBhQeCHiylZpM9RrQdYJjmflaPZQN+6J2NUOaND5iPW5fLiTW1YTh/Yt1dJqmiFgUBT+pD3lif7SuDDXyM5A6bxTU+bscuGbUnjTjdxVLq2Xq6geYp1HttmjtNJ5OMqWl61orCSrFEhsZ5ZqDiKGh/4p8gjoiN+FDi7qmJUeyTUp4lvNp9SXjgZZ//Dy/mi2eTXtO8AppuDYo6fksv0W/mZ6RCHxmJTEf6Z+Q2qMRiFUAMKbFSMLtlThZBkFuFCeo5MK1KGWPGXmTEXjgNibcNkJKjQDvXka/CiEqRQ6JnVrCLTSzMfE2oiiyPNTO8YrNGYNERFihIWOXKaKyvGTJ/fnnn0dR9Gu/9v0kwSxzaWqGw+HOzs6TJ0/Oz885M2dRFEm8mG56BnH6iNZxFM7RMga8RiIj2DzzTIOAlCMblPjhdY8see50neC4lZzKKK9HSO/S0dMjTsIcj4pTF0ih1YeirHR+8cBWq7VFWCnMhS7yzOrHmrqAVKPt4Yu8rVaUbotcd2mRXVpw13Ra12gg1ZAGNlsqmiB0K0oX7HYhtpmKqJazqDnkli3yLJtRHc/cVo3FThNcJh4CIldVi9PFnCsBiDdrADgxcQkv86e1LPhLDi80xiCStQuJgsoUjirvitaRRXzq6jUnTkAGCcEBGeDNKYYMEu/BQENIhICRsRQ5SOM4Mnkxn8/nXObx8fHjx/s3b95MU1NVkCRma2trOp3yIdkAIDJJ2Ezv0V9rHFvrf+205msPx8JzmqX0tUZkUNsl5cngIvhVmqY86/i+pJGdz+c6zADVwvBFt7d5h7o39Tpq313WVU+nlAVQaLsCjgUCNF28/qvrox/Qz8i2FEH/FZhyCeLFB/iIMT7ic7VU0FVtBT4BHda+df4arcDK8/pTv073sFwHn6w6iJJBKm1IMFNWLK1gyQAMmsSuCIIKzTJ0wZjFUXtENM+zLMtY4z49Pb1//z4RvfXWHSJAhH4/Ho1GnEl5Op0OBgNnFp2sK3kJ/lktjFuf18MEz4qAejn0OsAxqEGSDg20MzUZluqtHgmNraD31EItTy6ouRFIgtVwHIiH4GZXu7qoKTw6baAdxXRrkYudLHK4Dqgtdq31bA7BJdr1zPKhLvZ0V7cCzVckPttQo5vIJ/L7v43Px1YUBcftNvmwVfaD3wqPPsAWfSKUVhaiet4PWWgHwbm6AoFcVD0TdlEwFwIyBqQYLoP/NMaw7PaPIQACwGg0EmtenueHh4dxHG9vb4/HPQBwDthkMZvNONlQmixCmIzKAGfWzwnTbNFqCvgw0OdeFV17OF6XNINqeBXNRR5jYo4Xey4qG+VqMR5QUD6sI8mbRWnFkElsuwGte1q5MOUK7UxTsJ1E4ueeV2rjVpS/eLdfmuQMQHGsuXoOMBEM/GdzW83laIVQ15DxTGWw6br0sRPLF8GzsFi9mh+WdOE+cTsfuE3LYw3SNB2NRnLU3nw+Pzo6evTo0Xj8NX71YJDs7OwcHR1NJhPnnMFFfmTJjn+R+qysKoLmlg5OEen1yiFY07WH40CsyfQwHS47OXuN6ln7WiMoiMhVy/274DXiIDWqxL2tGFpUOjVcYBqs0BaxnpTuclK9C9G0GiiPme5MbNpKoC+eu+1YD0Ewi8RgsqJd69J0OkUV1MgONCJiV1tgO5YkUNhY/4rY1gxAyiIkWmHQY0Eb+Y065JHTSmgO159aVNQ5pDYd4AJuDyX7uYY8HRx/Q0SAzpdrACBJEra2s2NwNpsdHx9PJlm/n/Lg9/v9wWBgjMnzPMsM+cRYeKlciV0V9u3qfCzQjS43j54vvQ5wrKPN+GZzcqoZAvKM5mDbdowbEbEeDMpYyYtTtiQGqspqM9zzam8Ax3y/yyncZTvuhmMbRDhAfVttsz66TGqY5FY8vxYpUACoi09QxqXLFd4kzULUsAUF7xIwlUrq2jYrRnUtm6Ud+iAZ3V6oS26j8tKtBi8trevVWMbYQH36tPcDVAYjHR2M6IyJvApigp4pi2XwDwBUVcVJkJ88efLmm29yGHUcmxs3bpydne3v72dZxqHQLORauytoV9dX+pnl3OxqV2MTwwULf6F07eG4GXjEnzodtY5xYXhqKhSioWiQJaIkSbVuqIPGxP0itsvVsNX8qvWmfNXV5FZJ3qlTrMlgAY/qSdv6fFMpk5Cp9V7cQa0TRs8lqG9QfF6IHKChBGOxGGDDutiOV9jW0RCR43PzlDpJNrJVVdEi0M0RVERgEcW4hD4BLH9WFZ8+ZxDltC1XuYpcje3lU3KecGnC3sbUdmmK8OjsN0donUFLPqk+Ilo0ZVUCgjFk0Oq2z11prY3jmANDeSaWZXl0dHTr1q0kSTno7ebN7ePjnf39fY5pi+OYI+rI07rivPUniNgNyBct+WXSdYJjYSOZA+JvQR/co3FKhlYH/CJwFh6Mohi9iZPjao2xPBkWChEimkX6dl0g+qWrZJg0PpUE1Q0goiJxCL2GUeeJeZeDT0mlMxZtK/jkwmX+a5zS4c/obdw2Cu0GrOZ0wasIHp2tmNMOBLoe+VWFOKD0e9c1VmRZJt0l4b0aJnSvSnA0KM10NQXSRX7b9bweBe4NqZIOWUWfWJ0jJSSoXPokjdIin+tcEGVR9Ho9VzlXATBIgzVcIzKkTlGBxWCxzToCgqok9kWrypN2CXIFxOGB9RMLud84TCKOrW8FAIBzlVgMtGCL7AARgRCIDBoCAoKqImNYk11GZHNwxZaBPM/BwNawz5s+TBKfnZzE1h4dHibJ3TQ1REAEe3t7W1tbx4dnBZA1VZEf37lzezga7O/v9/upMTqO2AAZsd0Hq2ERiosZAUhuYe1GBLQSOQgExD9DRDDoiNAaMFiRIwQbRzZ+NXuChK4THHdRq7aoJ56e2Fr5ZZRkzpPZpeWzHnX9W+0FdvV0E5yKUBhasCkITZWJHcBE8MZWma/rIw/Ialeq5IFjPe27LGt5DMQ114S8JsBpxGwtHLoRsOkiC+TNS6ZAR2vyj9Yr0afPR3XAB/9QYJrFm2xvW6HlQf0MkUD+XZBYfkhN+KVcGUExUIgmfzK74jKDYJfWGbIrb+pjcwQqIn9O82w2m06nUTRixd9am6apcydVVeW5tRZns1l/kPZ6vVamxRUq/OtC1x6OA14RRmkGI5PfjCQcL4yur/UEgLY5IPpRK1uvqGcwt9GvgkGZO3QFdM1RWQBbyyevtlP9xFyCrtwI7TZlgPYmaG1dWgHKlReIsXVnju5GUMj+onNNdJEAoq6Yq+dlhjowaSebcAJ7tDSa80rIWtvsTKj3gFwEiKz5p1kCeP7XvxKlQdcNFMcGP1dzJ7RpSLcE0lfmF/g5IquloiiyLDs9Pe0N+kmS9nqxMYsYDID9LMuMBWvx7Oxsa3vU7/dns/N6e1va+FrStYdjCLmnhbFA8bTOYoVq/6guR0O5bM3SJWuriPAfX/P2AWrLkAltrsWArdGvLjWi6WmgJ2SzK/T0WPzZwcBdcBmpfd6gTNJ6NgZCTrpdYmNXTJsV4kT/2YSnl0wiX6UmTu111uMFHmR1V4hOqoUW1LPy6xfJte5bpoBz5GZXCXwhbjFtrNC+jeBXTh2iKPBdVRXiUvzoBjKjyh56Oeq0LHPwk4KXBaWryrIsiuL8/Pz09HQ4HCVJbAwkiR2NRiycsiyLYzuZTObz+Xg8LstSjGMrOuH1o2sPx61SHdRUERKU1F/p2dVFAQYFcyaogE73Dsr1J2IgUHP03IZG4jf97epKSv4BVMp7tTgrr4U6t+d3v0Qr6aBmSHD/cnNGdkME9NXDnl4OiZEKPLoJvzkVsCyZp7Q5AlRnyv40KTnA8eB+swSoL+ACIdEVn75Cgsq3q58JKmP8sXuuJHLEmDudTmezWb/fj6KFgswev7IskyTK8/z8/Hw0GjkV17+Ya/TsSr4GdO3hGNpsxwF6Ql1rQG9s5TvK6RzaClDFEgXsXtWPCpXr1ogoDVVaF9afok9R3TQRIGAgSIJyAoV6Rad1feuqdtuxDh5oCowANFe48rpmVNBGoa56vuiZGbxdr06ao6nFvMZiUPGCcg50kAMvANMmyOqWUt2cHTwJdf4U27E4M8RB0vwVi2en4ovMIsdQiz0QvaYsp3uIG4afFeaJosgBuQpYlWZE7vV6cbzN+fSTJDHGlFWZJIlz1XQ61c5z/dLLj+X1oWsPx10orINm9FQPYFcvKlsxPYhdE5Tp4g/JvBXMTDFKaJySCd90gutVrZ542tinZ3WgF+vJ1kpdtmOD7SwR2Ha1+CEVB6KN163ldFVJ23w03nXV/0WT9jGAwhfpW2oQ1IWTePytOuADve24tY1SsmvEBevy9dBr9ob6AMm3AsfSNFDRF6DSGeq2+ItQ5Ovyg/EyxhAtvZQMx4RQwgKsi6KYTqd8JohzaZ7nXIhzLk1T5xapmjhRTDD6wbteS7r2cAwdKykBKT1VEFHUE4k8A8/0gnT8EwFojYx1zmtx5TWd5s0JrGsIdSniOlLhNNWioNXOp0wkld9ghSuvCxZth+3YNFIFNRuobeXrTptgVfFMsfeiSfMP39FBYFBfIpA6kUC3nZMI8y41wWiOsuhql47JAS+YWx9e3TM8XjpfttiOWx3RPC8C87FzLoqSrhdpUWR8oNt8PtU1tNY6IHLI8Oqc410hWZaJ5YRLiOOYyM6zqXOu10vkq18SvZjp+sExqrBfPVSCofwM7x9thTxQaqaYLHRwMSjw1dOSeY7xLsgM2/y5/MnvJR+ZG2z/Ey2VpwFPXYlyhQb4iqIkZkopSua8VqkcOZmZiMinMFDbNrOgPq1Cpam1ofdi6W9b0Zkf5q4zfpcXeZ+nFpzgYYvFZysbPN9tJk3i7hKE1cEq/IA2vuvK6PtExKG+urfLsmTudT6hs7wCPGcKxslubL0lGhHZOaZfrZc72rKk9QzwZ+4RUbDnIkmSQA/gP8vSCeewIJEh0+NY+QOcYDF3QLOK8BsXwvXnHJtWHbE6Ho/R0Gw26/fTrvHSXa3ln3SOzNnFi1zt+AVo6DHg0zklSfKKjiJY0vWD44C6OjqAReieeEzCSdBQV4MnNeJcnJ6XuqfXlboanFFM319ceCUG1JwPDCa6/CYK6/uyJpDHgkiVphzCRqSKPo6aS2AVEpRMFTXtRQe6mY4pGOToCKDq4hR0l0ZJ/YB+hZZeGl8qT4ykopRcrkoBdflO9AWoyRXMC21rQkSipdnd+Y0qAMDmYz5hj0Ownd/dypAqtr6gu9YlrQBBfbLw56sKoFxN1x6OoQORtckPlRGta4ADHqWGUyV4Zl2Pf6WO8mwiV7MCXaRXmoE6Gdzna0mZJNXW3+qaCODqEoK6Yd08HXRpIMZEOxZDBytZEgEicamMMkH3rkbAryjVLk5aul+Cmmgb9LM8KQCt4wW1H0JjInoFcF0+1AOKyjMhgwINd6UME3h5LPs/m/wjDdJQLq/gNG95ns/nc461cM5JLrc4jjkhfRB2crl+5pvGByBSnTZw/EJoBbwK3+th6Hpe9D4NQ1BHeY1i685PjS8avzTfa7xe3V6s+3agLTTKQ+GiKG0U1g2EQEJQixAS2NUV0/0Z3NcXAfqLiQPVpsRmf0qB60ZorEsrymdqyq2vQnqIW+WiZlSBY1GH2VamPQTrVqDZz80G1mF0idegDkhrwqU3trALEcX2okl+KOk3OeaE2xVFUZIkfJiI9mYjLgLdVrQ3mLByU97bitdXja49HFNdzpNSIi5RTvOTKWC+5zicwuuoVJUV5QcZx5tVqmErAE/w5lSXhzU+ImJXz8ljEosS2KCDWQeK7wXyWD9CHz4ohkhsGCWkkM5MdS8+HlnP20twlBQi102OujjxPgvZ069DNVqf76qwxkRdEy0PZDVjjOFMx83KG7VBRqpERMaErGiAjKnF+zu/f0QgXvINSRwq1oOa8FkR6MLkQeuqjsDNFUW9QnpN4Jgaaq/AxAWNtvrMLj0PxcgQiPd1KXDNSeW1Vi73V0gUKUTLIWgzhoBnYpla0opWxvXVWF7rNwZqUbDWDi6alZc7Yvok5frj2K+gMqvh5nlRl+1YA0FdW1y7fN0bTXYNQE3u6GvnHGfLYg+eBqx16yPvCiqguULfbw40Kkdx0LSqqqIoNsZwTuQFviM4u9ymxK4C9kwKRqdpaoxhUc0COKzJhaWhtILr5urc3mT7K0XXHo617VKrxtrTKkwTzA1NzekBdeMa+AljVm4C7iKdIkdK0w9o1XiFLha4gJpqcgCIBDUPvvZ3Nx8GADmMCup9EvRkUELr8zpgFvxkkNxmov4I3MszpM7c7IqPfgmH/kplVo/IamoawYSLoM5sgd7g/K5l8qELoA7Y5rqZ7nzHz9SOA6msbce6TFHJ5StUodOk9r+0ige5LyzkFEk/JEnCYRtxYpuZxC9C0q5AqNvI6j9lNC8tyV4oXXs4XgGvmvuXsrrj+cADK9AjP2+Wv1Z99PC3QhhcDJFbN1uT31UlHL+svIdjMTIEvBjguG07OhO8f6kZ6EZ1k7fIKu3q0dc8vQNXnlMJSPl1zu/y4sCsF0ddcNY6XS+By8J1Abn6ziPNqHqk5FsJvONQxUBR+Cqk0T+oCaooRqkV/0Qs11g3XGiTsZQsQ2wiKxGiOk80a8plWUaxEcF86QbqYWq6uMUn+VU67QXRdYJj3bMSCBXYNOUBHmzwapS+biVO6csxWOz95zwmOjGmtm9eLsBI5ptRm/SErTU8oV8qyoQMkDSAS4Y5SXorvyoKimxkjCGHiGgQOWUtNHIL8d9oBG7qPh9D5BygMwbF6V85l8Q9vccBPJLK9lmJIGaSQz9F/iFikiRsR9Zgpx/QncD90AXTXRN43eknNm55u0gIqC97saFRatK5OHS+KrY5aEQzfhcfPyCrGb7mTc8SNQw+Ync+n4NiBqmPDivWXwV8q2ErEBL8ZxxDHNuqqpyTrACY5xVPPgDg1gOgtWhtVBRVVZE4AKuqclUF5OI4RnCxQUNVlVcRplEcA1IVV2kfrHU2snGycFEC9KIomk6ndnG+X2UtEFQ2suRWqVPNgXBlPWARAHmfIYFB45yLrC0qhwRpnCA1p8VLpesEx2uR1iygbWUdECq1VP/keZGUpmdys3qXfrVIqWAGss9aZpqAY5c4WVf7C3ZPCbIIDDXVQO0x529XeGkCF58Onmt9/nm5+ASS+E/dQP0MtDnHNAWLD6HAWEEqGEv0Yg2L7OMitSTSG1X02/W3siqSYoUf5E4gEvQASZpmUJv6RBjo5jQ9NMGF1k/rxpbQQ+hU3HHruHR9tS7pzny+JV+aXls4Xpe0qiJ3Vj+/bvlyIbMIGrZv/fAz4UAT1W3cfBFgFirqqudqOA5egYh8tGvgJxQ4Djge6rZU/ZOuKmHDbL26/s+LpJ7aoxj0Z9Cu1nIYFpvj2LQXyediXa/yBVdVpXOhaMEWlBOgv/6TieG1qRyI+UhsC6x9825mUd7JL1ACy4a8jk8Jab5CjBhQj9IxsHTGYD1Bxwsl3XWrOfBl0msLx3RZ7VhG5UVYlzQgCgejyrwDDbjRdV4No3pqkY/eD7QkUPppVzmr66+fQUSxXWi1DtRmXGgD5UCtW9Eu53OMyTjq6f3iSPck39GGiybqdfVbcyj5Ux92BfUuFYDTr2OLEPnNeNq2IBVAtbYLYpOFB2T3qRbMWNe7UZmDxdCkm9lETKx7EZqaTSCQmBsDOJaKPXOefnUKRnADx1eLND/JTHgJbLEudb33JSgUXe+lusuFu04sv1p1IhVRIKi6uqubSPfMoXkltK54aOK4aI7GB3QHamnQkxoZNd+ufm9w5qHIGxafgZvLLRKtLQMhpJyuwNAV3SDF6gskp4+/00LihdIGjl8qUd0XLNddnS5sjd6u+nzheIWeCPUVsVQ+UFhWvzTQkuRP7wxZhC7IDO8KIOt6hVO78rQWX1WlUwdJ2MbJkrr5gUamaXVXB9rTS4DjQMxgXX+UZ4Jvm2Tqnn35YYBu+trU99cEvQpeWQbF5PKkVEbOTpSx0EYGeV5qLkZb3cPaKAH1QXT1jUuw5JyagJEfovKESwncUsmrFXBvKz0v0NQzTivsr5ZeWzhelxpc9Zz130C7kZt6Juv5JsAn3+rFciuJZ0bPKE2gZkgXdZVP9RWDfguobMt69kJdluh+0B0SdELXe3Xbn6OY7CId3CJvDFRg/a6u92qI5DvcIu1KDfpHv9TVc1agMkdgY9+aFGJUkpCAn7Uo1SQjKC3FemrmQB5oaQG1YQ0lFpM+oy+QbUady66b/0KpyaLPd75fjl5bOKYO2/GKaaMl83MfngATZV5pb7WpZ9Rdi/Rk0/qpPlEYvd4Ez4KPi5O4mHSjnD/8oinbOAMvXDiyQiv1fEfe1VWf1vvrdmkTg4I4hIBhuij4Vqqhd1Hqx/ToiwYKALyTTYfKMcDleQ71fuZr2QYlvR3gKSh7Bd/kVD4c4ikAmud505UHq1JNhRJCCw890NKfXLj29a3o0hXqQvcgXBu6TnAs3K/tklq863kuGXP0/o4VaZzYlcxoZXwq5BVI0TXtVwSQ6ZrLAj/LMqhLi9UznJPnBpMBEYuikOlEPmFVVVVi+wM1JaSxXKaepTpiSfxF/DwokFLPL9ol8kA73wL8kqHRqIFeCdWwDn66Sj9LUcGfwWeXkSfoW6nbCkWM1JJf+lxQg7wWyReSILTrM7hI01SPu3QaKPOx5EQ2xvAmaf0TTjHczPWheyngpQCzAkkgaBuEV/MdYQmJ9yjLUo6A4hKstZw8SErweYQXe3+sXWy6k4uKHBfFbTR+E6Bzjo8fFH7TieWa1JRtWhg0KYqioig4yzMfQZLneRQlXczwcug6wXEraXMY1td0z6X8LmRc13XDQKkteqRsc8JAGhmpbpiT+S/PCxYQ1aKFoGMtpvE3mNvL+2u1quGjh+4e0z9p1nDd9enqcdGFY93E2fp8k9I01dCmJZ9uwjMbqyssfU5EvA2k2XUBmAaFSB308D0XQr8sExFl2rJttAb/aNLbZ3SxWvAwiJO32AYencvNXD3QtNL81fwhvDrPfEDXHo6pHn3JjGK6c1M8x/eu9bxWS1mh4E10ARzzw+hX8dCwXfKEkWWsVMPUAzadTzMfdIiYCJ3KRyyT3Bizbrc1YXTFXNVtDJTHpi4T9EnrW4KJp7UnKVDUtODh1SOoM/WAVwxBwX0Xhq4m3XxUm/GkPrKtptkDpEzJqzvnEoTqNF6t8Ab6gfTn6s3rQQ2l50UdkVagWTK5vGVdXQe6R/MirIh1a9grpNcBjkFND1E31h3RFdN+rftdbCHTTzNowAFaexKdUV4UOL6hzuvBEiFQD1Fp0wJk8rC2b6xLTUHS2j+62oHK2bwOmrbivfpPje9aJGgOCXCflDEkIC3YtBVFbOIBIq/qo0adg5931TwYZRkpLcLXpa5fNT0KwifSdg3HXcQ2a1m3McxZa+UIGEF8tq0VVambDM8SkytIS+UVnNP6qw0cX29aF45ZLZVEBMyCzKYXLEHeq23HrrGpL9AKuyiYWheZaV31CercFBVyf4Uiqaf9RWAu+Kr5IlFmW5tGilrLd/VdW05tuFhRjdZ2tT5GXlWUVG38ll6vp2FFj2ZrJ18avAJy/uhbET9ySIcm+XZ1aYFuBN5YJxpJQLq9lxMz61LQmTrw4xXStYfjYAj1rH5e5a91f3VRenZhh7Ob/wyMD80WNVUArV6JzJdiqaGOQWu+zfW7rQlqgRqo7wTTQGNcE25gpXBq/qpZ8jMBq8v1KqfTogrWtvVDoNfSwqCxOoa60NKiKByUOmM02ebitFrCGXV0juYNrC+5Viw92SkngdIC32yLBxU+xJ5AMDUOkd6+hL1C6wEXGRTdvVdDOX4t4FhbAISTXrSMXXcaiD6rKxyoYDItoTtngoRnakIVgaSD7Y3KGyclUJvpeVmfSx0hJkVpHIS6FJFZ3TptmtNPqtT1xtZP5zcXaDnU5Afde63lMxZLvIf2mgaNCuTNatKgozUJUnbhpliSqhqVxXBF5S9BOh22vFSno5MnL/hSVNRskfMHgkRJ3PzJJeA4YLy1pr/v0rVe+ELo2sOxsLWegbC+p76LybrG9RKRFeK745S1zqeG0fOc6iZOvTAUtQIUT8tvTX1HgICvrNZBadzBtK+rWutxZRPXgrnXRCtqi+9uusgu8mpoKI/SP3JfB+o1qYtP5vM5H2ysyzRqN+PlgFgumvHUEmygf4INm7hoxxcxGqyoRkDCkHp0WNXVcBz0dpMkaijILCqRJFr/gIaRCi/rVWtisQiV1T9UasS673z+dG3gWAvtoiiae0AFcUSpaS8H2henjtrPcKrcMuJSzJHOOUGtQCzLpuRAYSyKDABsZIxBQEdAjipAZyPkZxEAAAEYLoGoAgRjQeGjI4A4YVhxALVvnatsxGU4QJTrssqNMVHMSnFJjvhPVEGmVeW8ayUiIt5ZwCGZLEK0wJOZwx1CrhbeILM3yzLyR5/JYDm/PSSYiuDjvoEz5KrrrpkZyBIZNZZ2XG2OBeYNFAHoy8+7YLrf7/NAIyK3wjmX57lkLPPVAC5JElEGxIBu7aIERMMCuNdbWGkBltout50zBDkV2MtHGckzeZ4TEd8UxAxGp7Uy0DiiW4wJnMtC4JKUth70Nq8beHsRj6YcfQ0AUbSAdXHZ8XysqhIXMRUVkTHGxnFkjAGMi6JwFSRxAmSKvDLGVCUVOW+TQUSDuBxlPU/10OsMgpoCgJaGS5J7ZdCDV47I1waOJTBLB3tVVWUx3DqsJ9vFSaZl8Nuuopqnh/B9gTmthApaiZAg5RXRwB1omhenYAZSY9kbkK6wtvZw5nj+LWuCgYotWgwstPKF+JFGBcAnM8GoiL3mZzC7ApBdqx+kaVJycH2RyARqM7Wv+EnXV8KrqCLJ5GbwFucc0QLagl41fieFiD2ob3EM4LhLzAhKSg/zk81VFDbsIbpjg169oCq6Vte9/EJeOV0bOBYK4EBgWrutVtiOu+4HGcg0UqxF1CB+o95VpSFDTJOggMPVU71cnAKx1JwnzX4T77nc54PL8jxn1aw58TSCGLvodtmIpfGiKSb1bwM4bjZhtaJ3kR6ARodfkEglPtUw11XOBeFYd2bQA/7JyjW2RTB7sLKMiM0Em7oawRi11lO3SMSnBl+Rnc2Ty/m6a1enPNAc6BWkR/9ywLqi/y9R2quiawPHIu15WS3H1bTKZ1rHBNkkDaZdsKgPDNdsjfWtQeAZRXI7yJ1gtkvJX90mSEp5CcBUHtBRR7oCoivpb+Uz+Ikm3Xbw8afGbzcXRU/byvWnHl9pyOVsiNIWLeR0P+geXhF3rJmKlv7Vzvd2VUnepd+ODS2+We1g1ULKmCAvlR3nWmys6Dc+eEwrASI40S/snErkpnlGXir1Fz6BBhZrllvNzzpQZAPH14/EjEvr23tWazFMgaawbvlaTxf20kd/Co86lSMGGvP2cu0K5nYXydTVkxkA2BzJuMmHB4oKFkgdLkdHeoBa5wY4Ts8ynjwvMurkY8GmFX7XdavU9fwKbVT3iYabwKnlO60GTNLbQa4S53NZaLVgdU1WkxjrBZEBoCxLFleaQ1b3mEbt5s1W0vJDQ/MvIV0bOA6Ercj2KK7ZcPnhS4yojlK4yPNdtmN94KY8HChZckfc0OvWtklaH9EN0RNJazHNEmRK6IReTQimutZPrlaaPM+FyJ+8phEZwA9TfRmhlTW5vgT8aTWzJrnbbMddSB0MmQiY1odhJdvoQRdpLS61QHIHkE31hY6sNgSRXdsJ0CtATbscg6gJ6TRQo9AcXL5mMaD7FuqDcvE5qIvFlar9CrpeWnAXXRs41qRjfYTzRPFcePzXnMYyLZ858VYT1UnXuZXntNoY4N26hG2KtsCxYKjWi/V9pvl8TkRaowyswBqeEJGg5n6UwjU66EFZEdCmu6vZkGZjn3lf92oXdZWvpYL0VYcgW0X8m8DCAHWhqEGffMJJqhsoUOV8AK8g68gTGU15YEV7NTeSN/0bn0dFhBb6w70CIaE5BxrM1tXMLgpKvkivdrXrutO1gWM9ThKiD/VRp/rq+9Llg1IVV+zakh9qZnVq1wAp7c95Ah/9I0xPDVX6EryFbQaBFeW0wqsxJsuyXq/HpnkOi+YAtaaZeyFU0AhMCNpKu7T1MwgkkLcH87b5uS7pMqWl+o6rR3N39Y9UWMMxdMRlr4A/8ppsAGEyXs5noggwWsy4qEwugs5VVZVlORgMWk38XXwbaMRBeI9+I79Ixwtj3Zwd8LxUAxR3ffWZeEFaV2xfTbo2cMwkET8SFcApX/WWc+Hv1hKIag9gx8pOfq4DhgKsCR4mb3sVBRC8s4XZGpTVW6/1oLHKEyzTWpLxxH+ya1sMC9DQWcCDrGR1kfksWYZ10/jbwWDAv5IAYQmZkLbLu5xzQLXTHEQF1ttkBdqMz4qgZQC/GuuxWXpvYcc41rQwDXMceKBHVit6RmWzk7w2raQNtdAA90DMdME6b/8RlxcLOb1XQoolH8bAz4BnHmgLFgavlOhCAmG54lM/FoR4coJsthrHcZxlGacG5r1LeZ4nSSLHY+txBB88znJCax7MDFRPYucFEpRlGccxG6l1E6RbRHQJR9U48FIWZ0nnn+d5FEWDweAq4PY1g2Mm0VZQHSOG9X1Kl7NAvTjS+oK+udrLFHCnvqNngtZBmnKilXQifFIUWAPXVW30q11jK7ZgBykjiZ5jF6dgN6O0QizUUBeTktRRI7UgYJOkHxRwsEu2JXnImn2zrL+ISUGWOG6PF153ILqA+OK/Yur3+xpP9bKvtQRWOzTKcwM53T405OiGNF0zONZz2F02OBc6wHH1kwF1TQ+tMug7okEEn8+EgwAZA9gyPlO41uD05woKsB66w4wuMXOagM4XoqLKTUZG3gp4ccrzHBWJzq6jYgK8DsQDqqTSXfVv7c8VYHRx0pn86k1of/4ScrEVi7vK0SqOfj7Pc17QgAdZVu1XG0OkBD0Ezbq9QtLV62L7l0/XDI6ZeL0jhgtoLCGx22LVOgAreH3dcZLyNSgHAA0XkAdBE1qnUwDxzWqsIJ17QQjUkZ1ai7kEFogQampDYoSRWWrVEXAXpODQJu2JbU5+qtuvSGnuXbvXgu0PsOyQpaEWvgKs6LPyNDcStcPcuqT7X1d4BRyDUnSkPk5FSaM6p6oLjsUYJeY4VCelBW985SDYVJ5eLV1LOEafAIHoMvvMLwFeF6fWtdiK8le4kqDuMNFfdaE51s3HK96rxYaG9eC3XxF0XD2THPgDNwX6NZK+UJL4WfTRC5drVNBvfPMSEqtVnBDR87KxiT4LqtXQPZSaDbSTfDwey7KML7qyczCxmJcxZTeP80lIAgmEuPZhYM+LWvvhUnDyPOmawbGwC8vnOI4vZzu+IFY+89sLvqJZ1DPrqWe4NlxgfROUNNy0xfatQJzgvaImNP37ejKvRbr+orQGrnkBo3XBMQjA0saKQN+U+63acdd7g7hy+SyKSioPFzACdJEWQrq2zoXHEcgza5XfRavLCb7l8QrCZjjWYsX8YteuUfm2pLFNk8XzU4TWI+E9LYfoMum+nyddGzgOZk4r+qxFAgrPfO9axQbacQBtzc+u6REENoEHkSA2Q+6Ls15/rq5nU7NuNuTSKKC7NwBHHVmhM2asRZIDRACdLzhopEnOnxsrLn5E5Ei+FfUXah0pQeRL1F/SM4k44QuG+69OgQ1XKt81oME6TFqUZRlHPugKtxof5L3GZ3GDhrgKZORXbeRXpta17CukawPHQoG+tq7tmKmV854jae7X+Bu8bsWrdRsFUGTqBoC7Ata7XiFpMzUcQF1aNGtycQoqL355CSSQuD26lL1C276hHjLYJKhHVmgW6mqXVuieKUEvMZm1GNA90KV1rttFOlJQI/Izy8GGkJaRAh8Q0nqomNSTGvEq1AgEXFbplWrHr+bd3XT94FhmFyfktRECuEoFZiIi0dJqCXUOizEC4NTBBAAIYAFBoXNAFSx4Szb1w0rXE7paaJREB7uq8lkLAQCQN7OVFQDDRHA0WZVYMy9yi3jjxu7BwUGSJP1+kmUZOYKqRIDYoGiHiTVFNmetBFRoVxRFRSERWlVJJREhoEXLD/PLrLFVVRVlYYyJbEJEFtFGFjiLWFEBYpr0QUXa8d5oRxRFxhVqz4vz2z0itGABAQiopMqnmSZcnBYq0bUAtcFqHW75VszNok6KFBE1GRpLB1AHr2id2hjD8ad6ZKGRZ1nXIYoWzi42oUpy7W4YLbnubIWQa0Y0NrnEcWxty6ZkTau1eN0ovuZIFQZTWTlp8SM/5Bhhnf83EGakTMYMx9Jp0BBXelBAbTMxPl5e0rdy7znII2PQVFEMRFQUhTVxVRGn7QYIHa2EgNYALGBcrjuNflTjK631x4kFRGMhSSNA5wiMfcXRsdcPji9IQVwqyEhc6vChS1Dw9i7yaukCMvizqsqzs7P333//5OTk5OTk1q1bURQ9fvx4MBhU5QKJGM6cP3GS6otNxos8z5Nk4OqnPGitkNGEoWERChYTECHybjsCdGgIEcpqGYjG70EARENUETgC4r8BAA0BEWBnLgiZt6DmcFcvdcF08LxgzSUU1bWoOaDPHOJW4myZpM5b4usuR9kzYzp1r0LDmic9HPgG0OeEE9tCsKqQwrUKcolOboL15fptdflNwgtYg180z1ycXls4bvLipTtd//CZhTRlgMBEk8thuY5b2CJYCSpL2tnZ+eijj3q93s7uNsPu9vZ2r9ebTeeMoWVZsl1PNCB5uywtdZW00bksSw7L58nPG/D4AWxYV7jOOpJM94PeFyBfrZhmXXC87mK8C467YOuZcH/xUdbj+EyE6uoJ2U6mzUTPjIPuuq+rESwXghpaK6eQLO8EhubmK8gb5YPXXZzkh6J9A69IMOQERCRa23jVyXIdNQ2a0JiXr4ZeWzjWR3Whso1egi4Nx/LwCnjyh+ssNpiKqvLzn//8P/2n/7S1tXX7zi0i6vV6vHv1zu27g8Fga2trOBwmSdLr9bgcV89eJsteayPnljknAYAXr6PRoKqqLOOFOQBAFDEWA6jJLA1prvSlUbqxz+wfQV7xvMOlNrk21erV2tYFv3qmvibspAXJJVhLxIbISH2g1MXrH0hNUMb0poTDhuNhhfTSPUz1c7YuB8dY18GZS8EQ1hH5cnQJONaCyhiEV2XGVvTawrEQ1gnc2p2+FhxfpBr6fhcX3b9///z8/Pz8/OnR4W//9m+PRqMvv/wyiqI//dM/zbIsyzJjzHg8vnHjxo0bN4bD4e3bt5MkSZKE9Wvjk/RnWSFpgCQyP0mS09NTo9IIyLfWLlJV6G1+uhVad/4qa46AurZjPMf1bCt1aetdz2spFdxZ973QUGZXPN+Fmyt2xzVlKnS4aldUgHwCLPB99UyzSRfpxvId55zGYvoKUZXrklYvzOIY6VevHr+2cNzMQcPjbdfv9ECPg5Xu6UBJCerQJJ/Np3ZwBuu2H3300d/+23/7j/73f/h7v/d7vV5vOp32er2nh0eTyeTw8PDg4ODg4ODJkyeff/756elplmXs3uz3+wzTt27dGo/HiLbX6/X7fY4Mk/PWOBeBbB126qwmqG/fAJ8Hp9mWZrueCdMBLsj1JYwMup+fKRhWIyw04HiFNqp3r5hGorsLkmYnJkkJ1Pp8p6uq0QkKYlpsIPrAPY2DsktQFwsKjnXsxyUiYUQ1FvbWymlwfQk4Xlct0HrR81Ipvjq9tnBMDavZ4s8XLHc1TASQ0aode+ZbuHQEH0ejESL+/u///h/+4R/meX5wcLC1tTWbzVgd/vrXv2784ZXz+TzP8/v375+cnDx+/Pjhw4cHBwcffPDBj370oyzLxuPtXq+3vb29u7u7s7MzHo8Hg0Ecx8PhENXpUIgYx3GSJM4tF+CirQRudK1hSUIDUFhM3ZuPu+D4Etplx2pjbZeg/skzqyHJdKRYbXBve++q+usKrC6nCwEFpnXng9L+XD1LvaR/kwf4GXZCQKNj2YZGRGxVA5WzcC3Sngmta0t7tQp/CboEHF8F60RAry0cS8w/M5ZcuKI9XnIFXVDPbf6qCRZN8poCyXmUxhjE6LPPPvvDP/zDf/AP/gEizGYz2fVwcnKiJwy3K03TX/+N33Aq8eb5+fnBwcHR0dEXX3zx9OnTBw8efPbZJz/96URMzHt7ezs7O9vb21tbW6PRKE3TNE2jyFgbAxiebkQS9m+MiSRouKocn7BJRFHUQySeTbzc4yC4rqUfBxzDylMn1urerwjH65KO0wK/uCGiLMs63ttZjqilYhcSl2yTVsOx6MVNgIO6ENVx+uDxWjv3ggg/KVnnoLgEbmp81+o5wSV9g83y2794FltdZJK+NLr2cKzZSBRMqi9+Zb22ot9JBT+A4uA4iUkZUkWJAMXuokIiIpXLs+OY84xKNi8kf+Z5GccxgDNqG97Z2YSI/sk/+SdnZ2fGYpqmeZ5zJClHQYDHHVF2Zk+ecB3YUpym6TvvvPPuu+/+xm/8BhoDRPP5/Pj4+PHjx48fPz49Pb1///7Tp08/+uijyWQSRdFoNBoMBkmSvPnm28PhkPXoNE318VGoXDEsPLQbSprJ4bQaXGRxaozJy+U1G6zZxsK/Ah9gK9dG5XUEH5jF6ZUFRGToUend+hMarl0ZgkCRl2uugFYYoR4BEpitSJkvJITcORdFi2bKt5L/mhr7X4RtdGnC24Gk4T8ltIYrKT+XsGgpDf0RWVKCbkuapgE/CwNzUjexfjAfurYTFKG+vYhUvI0EDvEibNkPVQ5eqjl/QJ+IFg5T00JihdjWUqdZB/+VDvd8Bia8fLr2cMzzmYFAQ7OMijy5uLNm+VrdEEKVyzxYgkHdJkj1JWSTBLDEesug/41vfOPmzZtRFBVlTnrJGbWcDSjvkrPfNZ8ZT7u7uzdu3Pje974HADaK8iybTCanp6esPn/xxReHh4cffPBBnudZlhFRmqbb29s7OzuDweDdd9+NoihN016vx5nIpVHaTEnegiGdL1jMIFVRST5rPj/DUMWDWFUV4xT5jQPS4YEUDJQ4eVGXdhlAMNbNAk0El5RD+ufQ7XoVI4awBKjgXy3JwMuAQMc0PpwcvNSRugXbUjQ168nXetfiJZTZLmp2WvAKzfl6DgY9GUxMXRSsxNzXm649HDvndL4S8mdutqoSdClXHtQ5T9gO60GUoEx12O24CEjDsQQUl2V569YtVorlIAY+4xnasAM60rEDQBzHev+brj8A9Hq90Wj0zjvv/OZv/iYrcfP5fDKZHBwc7O/vP3r06NGjR/v7+59//umPf/z/474dDAY7Ozt7e3tbW1tpmt66dYv1HTmDg+np06dRFCVJwqt76Ui+Zmg2fntbURRsmtRbmUEhUeCYcj4OBH0G3me61FbbWAWdg5EKzOLgI09ax1Grus7v6JGcZYE40eqbHhc56UZLONcWSydCCOr4pYGvqZSsS1q0tzJeQKYe0SEliE4QKKSiymhxdQnb9OtB1x6Otb6jOT7g/uXza3rwm6KbL9j2J543JmttUS635wojPrMJwpRywetxvWTWlWl+SvPDuYfEUwAAnHNlWbAGzYtT8IobR8UxtO3u7t68eZOV6KIoZrNZnueHh4eHh4f3799/8ODB4eHhxx9/PJvNiqKIoqjX621tbe3t7d24cWN3d5etHOPxWLfLyc5AmwrESE+ypJHIVvDoJkmHncrCHECPXqCs7mQZlAAs9Cjo5wPgXl2+LlmqTUTWYlAy1EWCFKvtaXJTKuk6Nv1LIdq4ETBDE/0vTl1WAqlYcL85TBqOsY1023FpVfhlpGsPx9YfFyamTPIZzp5L+cGc0WxEjSAB8otowRpUB012lS8RTvwMg1eapgsOXmybrhbNxNrJZvIpwB1MRVE0uCZpmvb7fUQ8Oztj/OWpnuc5nyEd1bU/1nCHw+H29vY3v/nNv/W3/hYaU+T58fHx/v7+8fHxZ599xibpzz///IMPPiC/Wr9z5w6DMqvS29vbo9EoSZJ57pxzZVmyUmyMkXPSxNBJauuwDEHQpeLblOaz9JJ9MRck0bJFO2Ny9Qyc1LBvBCR9TnVL91qVAa9d6hJQBcDoF4FfHGhniTyvs3a04vha9YH6AnHF81IxVHZbbGjHqDQMdrfp0aTnZ125XnTt4TiQxoJBgQ7Veq2pa6ZpLtGsJvl2ZaosThRVB92j9+9h9yZgRKutz7x+d86NRiPvAFxaGxEXGbACxQ3qs1S+1Z0TzHB2CcqkFe14Np0b7xGVfdh5nrvlNj+bJAnblAHgb/3e77mimM/ns9ns9PSUTRxHR0dffvnlZDL54osvJpMJS5fhcJim6Ztvf308HoutA72hU3x3cogcp9Fg2G312ulGQSMqYMU4ajYQONbYgfX8yFADo/bytTkFVfo9Pt0j4Eb9lqDy2JbrB+s2DY1orhGUQnUPoW5aV+esoCZHBaXJtX4Slf8GfNAIKh1ZPy/1kgH95cRieA3g2PnILVTLHFRuPaxrE5d4RRP15L5ciI2YXA21YWW6MlC2Y/ZJGh9NLKt1GxlEZJxyzhV5Gbydr4OEjcG1qUdQkTriU+vX+igHmUIybUTBYT104YI7P+fy+/3+1tbWu+++K6XleX5+fn5ycnJwcPDw4cNHjx6dnJz84he/yLJsPp9XVdXr9Xi7ytbW1ltvvWXtYscKm5s5Yx9HCDSFq5YQxh9XLAb0Jgk/SCHNQdEoJodt68cQsSzbxarUpFWr1Zwp8Kr5SuR3c7sN1fdoBIgsR4XqckCFiNBX045l9AMZ/0w41m0kdWisliU8T0VlCaIy1q3qa0CvAxzzVAQf9cV/BvlwQfh4TVdeky24GKvOVNcaQVFkwbqMlKeiSUZtxmP9FADYkCp8LNCcZZk1IezypzaG6FnHIC6BVjIrOLZBQEQJs+WxHVK3siw5lwJfi34k/Yzq7FGtAXHM3De+8Q357dl5dnZ2dnBw8OjRI8boTz/99OzsjMvp9Xp7e3u3bt26efPmzZs3d3Z2eDtM4MpjccIuygDUnjmOAXzwKkELVAm3AiXFn4kRLEG5MmI9IGVQdirkRvpfCwCBY1N3BZP3JeiaCwX+Mf0uaa/wZ1flV5BuNdX1Vt2NWh4075BaeQTlO+esXUL5ipnyy0DXBo6DgRRG58UsWyGtP+M2yzIJjEW/1W3BE+YZNmV5mF+axgkoCzI5IlcRw5zHYSMablmhD4xFdQLFipngyirLMmsjYyoDOJ/PrUVrrY2MiW2UxoTufH4OADaNpsX5wPaOj495o12v16+qajqdDgaD2WwmPUO+ZkTkXLyoJRgAQEAEg4AC0wBg0JhF1DOB4WBPXIYrgDOEZbU4ksdao2dahItgD4f8aiwcAUCSJA4BHGRZLWPk1jjd2x1+8723jTFlWc5ms/Pz8yzL9vf3Dw4Ovvjii0ePHh0ePPj0kw/Ozs5ms9nWaDwajW7cuHH79u2bN2/u7e2Nx+Ner5ekBsAQRaynV1VJxEk+Iy9RuIaL9xpTWxyANzsYRC4BADjZaAu+Ey1sRgBIThwVogo45yJji6KofHgvIBgkg1QQoLEGjWP9wBgHWJYVcSozlWCXbagBfDMFkkBfa0GrPyt1NKoOe+8ymmlHoo7R1id8a8TXsgeUhOD7RVHwSSu9Xo87iryOLx4LD74JEbL2HEWJd2bkvo2LT2kwUsduz8oBAILPqEncTWxSJzXiylNKhgisiUfDLVfBQgq8UjPJtYHjLmI20gMMfvs/+IUe+eWbMWbdjZFaAw3uN28yt6GyPIoKs2IasCBxTioJ1lrjbcrgyySi2TT7v/7f/i9lWY5GI0lMwVENe3t71lpxi7HllxVS0aHIb4dlY67UmXwgsHMuiiUFPoknTeNv0BC7XHwAgiVDq5UwzsIs0tFaOxwOx+PxnTt3RMQWRcEmjpOTk/uffX54ePjgwYMf/ehHs9mMvZFJkty5c2c4HN68efPWrVu7u7uDwWDBA7AIXi6KwnlwRMRer8evJh8KCd4Cwycugha6RNw/IS5Lqz3AMYNVVcXBiCBsBgC8UCCHCESL2W7MQgy4jlRWK1zQWsGUEQmOQBW21EOm6y+7AZfmNTXWX504d4qUxvGLzgdxirBpXW00J9QvG70OcBzEHeMFYssuTl0TMmB6JvSGBVmuojKZtZZf+s3cVcXKwiLUge0A8/kcEdGQtXY+n5+dnb379W+Mx+Pt7W3+dv/g8K8/+ng2m3GSIM69yekptra2er3eeLQFas2LKtM5+Dks+19F15PnxfCiD+PRjaoy1qQMax7GGAob6gCWw6F302lkYfQkv5dvd3d3d3eXiH7nb/42m6F5V+GjR4/u379/eHj49OnTx48f//znP59Op8aY0Wi0tbXV7/fffufrg8Fge3ub00OznijeSLEVOE/zec5iTCMseGs+iy4eVjknV5s75eH5fC7dCwocvREBkBZuKwRAAtOhhmHHgRROhYJpXpI6BwxmVf5rfRGMo1Z4W9/bdb/J+UwSLQPqoCZSu2OErxAxiOOEbu3nl4SuPRyLVRHq01tWlLwqp6VFb70AuK5FoubgJjSLVBA50QXHgtqMUA7IOZNlWb/fR1ymWKvKRZKXP/o//D1eAJ6fn8/nc/CK8Onp6Xw+Pzk5OTw6/vyLL0U3vHf7Tr/f397e5mgzzpKsV6+MOHKwSFHmuifFxNzr9QKVih9AcogIDd+9nt+kliToT42Sx3gbHtvKq6piBVZkQzab85/D4XBvb+9b3/oW14HbO5lMjo+PDw8PHz58+PDhw6Ojo//1f/1P3Gkcy8FJOfr9/o0bN1iMcdpo3nZoDGd/tsYAZ49An8vfGBPHFjEFD82M6j5e2zEOLyweBnq9RC/UyrJk+4mJEvAWMOEE1x2IuQL+mgymFyuoYjNAwbEMFnX7GLA7EHNd4lWCoLN+u74j14Gyv6IHfhno2sOxOPGMypCwAv7WpcAjr+83b4qZjIFYonqdc5JrIiC/+WIZGOScK4ri888/37t1M47j8Thl/aLXG7z99ruD3oibORptiYpXluWdOyUv0lmRzLKM7XcPv7g/mUwePXo0m82IiDNwpml648aNra0t3rjR7/dZkQQAwIVBQxRDXnxMp1MRMIxWPOvm0/MlNLNdmhqqEzjwVkDE5Vml/BMpnzVQnQi0KAqx3XOo8qJARM55tLu7+7Wvfc1ay/sJsyw7Pjk9Ozvb399/8ODBo0ePnj59ev/+fe6WXq83HA6Hw+FoNOL94ltbW6PRlmhworWBEufG75EJcI2Ud07klvWHXQl6Vt6JJZ49XLl6eyYYXRCtAjVTKs+CULQTEa5dVeqaR13aMUs77ctZXWHpKOz2Gf7y0LWHY22W1eJX2O652I6FhEvIxySA0o6FpYxKG9TUnTUlSUK0cHEgogMGhejBgwdfe+8bN2/eRDTOlQDY7/eHg2GeE0MPo0DlwNo4Qgtok9QMPJTw8ryqqq+/8zb7NtlpNplMptMpw73k+mHH4N7e3mg0unPnzmAwGI/Ho+0xEc3n8+l0mmUZq+SIaNCQg7KqaHEQqkxjZPUeEYkWn3JbSJREGT49jqwac2/0ej1r7XRyLtAvT/LD4oLjxDScT2N7Z5fREwCyLDs7Ozs9PZ3NZp999hknIGWk/vjjj0XIsauQ0bnf77N42Nra0ugs46i3mQgGMQCRj6YgZTFDXIhk5g3t0rgIv+l3BWglXC2/cmrDdFDtLj7UTNv13hVfadzkC7ZrsTDT9ZSJKSaLxdsl4klVcgPH15XEpmbUzqXntSUP2iJeocGIGo6blRGobS0fgcpyYVyL47giVxSFtfadd7755ptv93oJr9n7/X4cJ2VVFZUzxtg44jJNVVlr8zxPolTKtHGUQDqAIQDksykADL1Ng31c7OPO83w6nU4mk8lkcn5+/uDxfvHFl//lT/9rHMesSLL6zDDdHy3yIwv6lGVZVS6xS7MD52eQVhs+kLWtPwWFyXsRZbsH95sEDFifGk2DsoY80eV9h9bYIEmSe/fuRVH0ne98h3XkLMum0+nx8fHTp0/Pzs4+/fTz09NTNkMXRcFbEFk+cSQ1W+E53Z219ujoCBE5F4esyp1zOzs7sr0bACQRkvUpO1x9H3AXHHfxSYCY8qfuSa0RN+OXQRklRCEVKNQ25YvUR6aARnm5KWY66RNURxzwqPl3t5T/S2uveB3gWG/2FXfKc7QdB+Ar94MnBYhl+pEPHlixPYGtpVxPURaMMe+8887u7u50Oj0/nyVJMhpvA8B0OgWbxGniwFTOAQGgtXFqXC2buK7eYDAQxYT7h5GOQfDGjRt8X5k4sizL2Ca7v79///59AGBRMRgMeI0viSnSNKVymZ/Bv5Q40gx8VFG9m4yAEfp0GYjIgXq8qpDgECFe7vB9lii8SUTWQPJ2Y2tdzY3Nssz5CBYO5HjjjTd4vLIs49XD2dkZh3B8+eWXR0dHT548efRoziuJOI716iGKovF4PBwOxYdclmVVFfwK9vhZi8ZEiFiWFQIiOaQKAJAIAckt/mwS4qqcKk1Q1mnjQSFsc+cIX0tkRUArYLf1fhexYsE9w52PKvCJ6jEViC2mrV9aLAaO0HzVdbgM/ehHP/rkk0+MMb00BrW5lrkzjuPWbSAAEKERFUzjplF78xWygECqjo6Cenp7UMY46zN7iRMPlO1CNCMRG7hY1xOnXitdVZblo0ePvve97337V757dHTU7/eHwyEAIJ/6rrRgTV3ajXXtB8XLbjfdagBAJN41x7YOhmk+jmQ+n5+fn89mM/ZEMe68ceeNnZ2dW7du7ezsJElijN8OU+ZsMa+qguOL2bc2n+et9TE+eEPbWJ1zFg2DJifxmE6ncvSf7nYQIwbUkOWZaJImPahjGd8/PT3lQ7CePHny5MmT/f39/f19PgSLrRyDwYCzJvFu752dHV5SiPhntimKRbYmTrdkjOHU1ePxeD6f622Q3F4wUSv/UH0jiTZbQ5u2qz2ugajWWguqqHyxlXMPCD/rGaFfIfxMSlXXUGvqkW2S7FiS/wFAXlbGmH6/zy2Koohzp3QNnDXtPpguavYP32exNJvNdnZ2bt++ibgIQNzEHT9Pkh439RybS6bpCCTSD2sBDvV0BPKJiuRXz6yesKasc8lPG4EV/rMoKgTbSwdxlLgKiMhEgGiLDtjtfCO0tzdOegEQ02KBjzZKkrQvIopNAW+/8zUxSed5PpvNONPb/uOD+18+nM3+W1nlSZLs7Ozc2tsdj8e3b9+Ok2g0GsWxZbsHb5hO035Xz0ivavfpbDYbDocCxGy6PTk5YdOB7GMMjNFB06B7dPI8D2Ysf25tbY3H43v37nGBvOF7Pp/v7+8fHR09fPjw8ePHT58+/fTTT6fTKZu8OYKFzTu8nzBN0zhOeaxFihARg454/3RMbl6GVnW5RmUfELQVxVOzqHQpNdZ2up9155Ai/VUzwSn5jF36seaLrgVdtdpeYzjGuoda+AzquxWCaaahRz+g2ZTvy8SWmxq5muZp/dtWEpTRWgOo3PNEtNhSRIsURezOKkvnnIuM7T7BsnOJ1/UDDVv6gu2/YuMDlU9OiG3HnF3o17736+zum81m0+n0bHKyf/j0s88+q6rKWEySxFr8xte+/lu/9VuDXt8Alh3bH7Tqp2k8HrOezrEf7MMcDAakTjgGH/FtrS3d0ngS6G6t1Fzsg4/wA2/3Z2N6kiRVVb3zzjvgfXGz2ezo6Ojg4ODs7IxPV3n48OEnn3zys5/9DP0K/e7du7KrcDweA4C1djAYVFUVx4zODmTHHzkxpgVwHMCorm0AoK3t1WOHWAvjwfo2paDHmnqlhv6L9PAVJ27ZFYHl6wTHgQqgF00y/eQBWfkKLTSvouzCYrmjeV2/HWqL+tozgfbRVX9SS8JFjIGfJIv5gMibJthTZG2MiFWVCXp2KPfdiyz3DNdQUGGxwus5bxb7BpfhyXEc9/t9IrI2HRTFdlFy6rKqqvJiXmZ5VRXT2fnDhw8/+utffPLJJ9/73veMMVk+i5NBezXrYQxSN7aNMCbqjb+g8iJx7y0Or0r5dAxAZIZZdE2n68wYWNg3ap++vWWVlxyLvVAM3XKvYxzH9964+867b7OdlJ2Ek8mEtxQ+fvz45OTk/mf39x8//uTjj4kojuPxePzee+995zvf4T40iItTmkRL6PBtyOJJayHCRQEig5oU8qd0qXC4jLKeOFDnap3XG7zYE3GlRwoaU+YqUzCRAYDo1YPydYJjqClxuEI7FlHfWkKA3VDHVs2dUGfrJn+vRvbWmksFgubocqy1Ozs7URQDACAkSQ8AiKMjXNcRme2sZDruaw+7Jh3bK53MFWb1UxQ03hyMEPV6Fvt811VVVVZ9Kqs4tlVVDXr9g/3HVVX1+/15Nr3IdCVv7mSpMBwM4jh2zh0fHxdFsb29bYw5OTnp9/vg97CQOvwpLzOoD+Uz39iqHc9mM/QaN3oDq2QgEesn29M5P4Mxhk9XeeONN8Av0aaT6cnJyaNHj7788stPPvnks88++/M///PPP//8e9/73s2bNzn3NItnHhGngrKhvqKSntG9FPBbk/20DrtCV9AdFWD0RfoNrhUWQ4gkr7YuS7qucAzLyVZbOgl2yL4vwY7FTzpi1zSyN8cn4GZ5Y8DfKwZWP6ndLIDOWOCsxxLaFcfx3u5NQOAoaY4lIwJrDZSr4LWFTPsQt4oiACgKCZBiZYG7F8tSNlIb3sltLTnnEPyJyK4oK4dESdyLe8h6P+/lm0wmFtEVZWzs6mWtoIwo4w8fPnz69On29va9e/d6vR7D4u3bt6fTKXnk0m49RzUjlVx3IxGRsnvKp5xBJz+UHdLg3W4irngfigyE7KBBxL29vd3d3Xv37r333ntf+9rX/uqv/urTTz+dz+f379/nLTly8CCryR2m/uWR1agMbq6+F4kaVguqm3T1BdUNyhrlta5TFAWqwDWSvN71beLQMB5ecQoUMn/zFSvI1wmOhQvruk9t0S2KAHuEm781jVmJfuUbPCzXgSYbPKk5eHXl5RkdE2qYiY1ZCBIAYwxPVADI8yJJY1cBADiCKAJOdNkknXlLU1eqGk0akXX5epHBUNjsirKorLVRZFMbEQG4CgAMOECaTaez2WwymZydnTF2pGmad6xbtAZn/KF5VVX9y3/5Lz/++OPt7e1vfetbHGFy8+bN9957j8PsOBwYESWeOokTKUfEHnUfacqksUnflCaz25CzeoISz/LDIDgPvKHs4OCAvY7D4fDtt99mlyDvSeH0OvIi/omr73WWa2GeQHw2fQBYN1MEdyRIQ8onn8ATFEwLybnOUhooZmZy9ZDqFf18pUhmrv8T4BUHVlwrOIZ6bANwV1LtK6GmK2/R79XS1qafD4KBoD5atTe2nc/YrECz5sKvpDwqnHYHYZlEwviouLKosqwAMGVZAiEhWJsAteNZ0rEJe5a1a82V88YKYkUYOSDfUbnsK7O0HTPcI+fQQCTJ0Ww4iquEAowFy2CCJpudc+ZMAIjjuNfvn5wcVVUF2Kmta9kmXsTBYPD7v//7b731Fh+omiTJo0ePfv7zn4vQGo/HHA29t7c3HA57vZR8KCFvD1s9Ll0wJ0epyvBxhIlWTkVyoN9eQT6+TdwV6VYKABz43Ov1bt26devWrcPDw5OTE7b2CJbxxsLzeSlsIJU0PoYX6qqrqZ+4Id0YwKLm9gCa5U+j9hPqt8g2HK12iGocdN01guNWAfzK6TrBccB8vCTs9xI9e2VLBQBUVaW3A/FKMzi6VAoMVL+Aq0zHr1rvy01Ux7PrJyVol3WuyWSCPlNEv98nAxhhVs2G1trYoS0NVMaY+Xz+9OR0a7Td2jlVVSZJIm8R40yaLHfN6eczx5EDiy8RvGBzZKOlN88Y4yqXZflyBtICisAROLJJ6ZwDoAjB2giQ+HjUbD7L5rOiKM7Pzqy155PJYDDK89zB8lRD55wjNQRokAPhjSkqOJsWT56e/c7v/d2vfe1riDiZTGRXNMecTafTk5OTo5OTLx5/WFUfcCfvbQ15u4qkTEqTng4kkHQcC+MDLVyIjpwYi6y1g/4wy7KyWBiFK7fYOQnOGURAQ0C06DckB5GNq6rCKALO0kdgCKqqxGoWx3FVlVzyzs7Om2++yTtNpudzcsiyzVqTZSVQhYYMgnNO5KUxBpGqsiTud0Zvg0RUViW50O2srwVJBT0lo57SCpch8yzJ2Fg/n8/ZQmJ95mtXi6cubP0Abx5WDvmX0tgCIyZ4/Tq+rtxy4SLJaTWjBqAP6PTuR7kW7R79uWJec1+aUJwK1pYKA4A40V85Pl8nOIa6roodtirhGFAS28mpCh27niRHjPBKV+gVU5e3kD3R0OayQ0XGZyzTESB8UdLirHvwNkq23zEDiastIA7D4n3AXGaSJLxMBsXT0oeDwUAawiTKEXeCZN1kPNKrAVGRoigCkN2uxFONlb6gM5fXiuMRF6ngfD8vTP8MGXEcczbnwWAwnU6Hw6Gc3ME7BXjW8UYVjrQriuLw8YPT09OHDx9yxktJmfT222/z0SQcDiyLoSiKeMM3o4ZstOMkOOgdDxqPiBxRKdjNncz6L2/qEVt2VVVxPSYMvctR84P0xgp9rRVwERdHHzA1NVNTjw5GZQXW5ciYBnYSoWYhwifycHBHs0pXo+SHwiGXtj6zHib6EEuUoijiKNV10CsAWA4BXBGd/prBsZD0O6mlGdU9Fc2vAjhYUbgglD6DTlMXHIsA0KAMAGzilMIrfzK0NESmKDliVJUX8WOIaCPb9V6GbPCziAvPskze25z8fKHnJ6gMkLrr5HkBGlmP8wEoiGhg0S624UooAmOZYLT0v8AT/2mtdW4xe1nk9Aej3d3dOI7zPGddjLOFkT+ZSZrD3jPWnfOvvc2bCWez2dnZ2dnZGdtMfvKTn0hoRJqmnIxiOBx+/etfj6Ko3++zGdo5l2VZlmXoD+vTh1HprZjCYxL8xzLKqFOcpZnSjVYdESACGBQWC8NoBIEOOAYAa6z+VVP8y7AKnpr6ng5U2easyrzK6kLA2KDAWldV2APVUlVa18W0ACAxKuQN6PQsM0JrV0htObXs8qhDz/igxBWpXbjGGIbjV64awzWCYz1ITh3+Fnwl/CEhUML0C9DpEIN607B2AXVxRtf9rqWWzGRQmyyMMXzGWqCbcAO1cUaex0Z2d6Ysy+I4ZhsILxi5FYsE9r58udanhFh1fAN/K8wtkgmUpAkGZTG9XVX5YziqqnJVJXDM6On1fdKN0o2RCYk+o1uaprwpOU1TfZYohyGLXs/aEDshXRqh39dQFMV8Pp/P50VRTKdThmlOa8cbN4qi+PGPf4w+P/LW1tatW7fu3r3LSC19Jat73k2nRRq3nTswz/Mqm3OrbcK+xhR8zDL6jG5ybojgGiiA4NzQ0s+CI8EdXQeNMgHf6mEVxhaBrY0PWrHV9g2d+wLUXENst0EHtXqmtisTs1n5VtKNDThHOyS5Lf1+P5sXTb4FP8vguaYb++p0beDYqW0CThGplZTwazBUmrp8p7I7Cy4QJgHdfKajJhBDuxUoPmam0dlVnMq3IloqmwsWcgWx6+hVfoZ368pMMMawohcgslRDXF5a1Bmfx0dDs1NnqUltHRtS+Y4/PoMlgSRsZ6WYk8lZaxFrejcRaMjjivGxJkmSsF6MiJL8XtYQUk+pD7+oNxgw9PPD7Oiz1k4mE+dryDKD63Z0dFRVFZ+0cnp6ur+//+d//udEFEXRcDi8ffv23bt32Uk4GAw4f4hTufEChY6hduHWK4uyquLIUn0RIHVGtT6QcgiWeCeDuwKhtOpAdesQqlWO4v/mjFgUIqevck9y8o1gNgVSoUkalKEhPL46kVoBgxJXbM5mdyj3w2w2K8uSneTNCqC3/m3g+DIkHUp1gQz1gQGlskHDdlxVlY3aIxBkzDR4QV0Ia+qCY9lcq0sDpUlB/YAiziMsh26QzzknxgoGRFZSqLs+rC1yyUQ1nVo3ChpzW4BA2sWQysZTaCxadaOcc9aahTmlLLWNwqnYBsFBBl5dGYCacodeUU3TlPzBa9wixj7OWcGIL2ZN3TRrFhooeaPQwoYYx7rmDNllWe7t7YnYYG2a8zuzrePx48eff/45H+y0SDq6u7e9vc3nqG4NB9r9xYMoARi2LeE1V4DrDyG4cAqhWkykxlDdSxqBQSEsKCuQLkG+sio1vuYNpza5yM+pfl6inmJSoMZrVDqyZrYVcCwqeTDvVlCzVxmOeSHFqzE5XAbrBwZpBnbqEPqrYKmAawTHetnFzhBB5IBxUcF0cL2ifEEcrON+F/x1lRbwE9cKEVk35MkQqKiB9RAAWLaD2py2ULoNdhlbyJsmrU/BxazZbJfuq2aLZN0qIIVqcxqqk+S5eqyss51CS4JKGSvIu92iKCK1ewKU6cj4PDucAL6qKo55YHuFVXnyeO7pw6JEQwSAvBK7isHIRl76Gp8WnTs8QuT4ZD7gShpVlmWWZZwmSVRpjp5mG/Th4eGjR4/+4i/+gk0o29vbt/Z2t7a2OEsyh3MwFjC4QxskiSlJQNk5h2idc4ThjjiNyJonF6xVtxRrsRoAsXS73JE5xX1Sqaynxu9tIWXTCIy8UqYuUPQMrTCtMFlI7GDAWl2kxZK++fjxY1aHnzx5goh3797lcyOrqgzEdlDtZ5pTXiZdGzjWhOp4cOiAWgYCqLuJV5cJynypBf6K55tk655uKYddFuLZEOQSXNa86PxhbiLGec1ubdQlxyWYAZSdpCkb5Fq0HqgrUKRsJqRcmqKtiynDRwgsN0wHM0QXu2xy/T6pAUJHbDImoizL0ETsWxPbMSuwDMd8YqlY4UkZRlvbC3VIQh/JJ6IdfGTkYDAYjUYME+wOZZZg9yBjepZl5+fn7Cd89OTgwYMHJycnAJCm6fbWeHd394033rh79+5wONSMJD3MejTUERaxU/br/gxUEMSaHVmPryCjbrseYkEl5xyvroQVrfdJ6sA4q/aLG08yvpq9pXCnzhBobZeeCM/E4hV0eHh4fn7+5MmTP/uzPzs9Pf3Od77zjW984+23337j3lu6Z5rl1yfIK1aTrx8ci9GwLMter2fMQtdI09T6U5DRu01AnQa0YM2OHDyVOm1PQBkaK035FN4NPgNYFK5N07TyMf+g/M5iopXfWgTZoQtefeCY4rIsDTzb1KXFwIppEDws8wo8ZIASY7wY5wW+hKMCgPNRH+Rtjtx7nCWZyxFULcuyoortBosOjxZbhPM87w2GYplNkoRgEdUXRRGnwYzjWPpqMBiIYaQ2n9UJJLrtlQpWZeKmifMH2rwOwQNElOWltLEqc7YgU1mcn5+fnp5OzyeTyeTBgweffvpplmX/9J/+06X1xjsDGIvFuMSpnOM4zvPSWlvh0gijBYYcx4VBMkyDTnm2BUk5H6lE8pKyP8hcqFR6eJ5WItKEASRdvVOxj3wzgDZSQSZC1m92Z3uO8a5sLqrX60FRGh8IBD6DlVFObN35zJ9JkjAfchKrk5OT2Wx29+5d9tZ+//vfj+P4j//4j/f29maz2fR8/t3vfpf7kBnJGMNZW1nAl2WJGHP81Cs3WVw/OA6I1OlbGiKFLihvRXEANY1RHWYjWkkXo7QWJY/JFgZtmjDGiPONLZ7OOWzkhbkiJPrLBVeUWirAUr9bXEsvkV8z8pwhb3UBXFo5SVn/+SdymBPUVwPO1fyEUpmutVFVO6zIiLKJZrl9Ro+ysZXcd865qqiqCqpyNBrdunXLIBDR6enp559+/MEHH5yfnfSHY+k93Tli5ZfqsUVonmfcNFkH8LUOQQMlP7T41x0lAY6yLhE0D7qieUeT7KdAFctv/OlKUFc80WeA0zXhCohupIWNq4chXYSstWxHkp4xxvDZu7yyuXnz5s9+9rO/+3f/7o9+9KPf+q3fevLkiTHmvffeGwwGs9mMxYPOBOBZ7ErQtYdjHhjxqOg1+Frd3NQ7tPELGq7wrsKx7kiR37q6h425k9UEqpvn4EruNBWUlJ4Bnl0+cE10KKd8/YEO7pwjWMar6r7l2DU0Brn3jAFYGlukZNH1AqwHNehYX84/s2nNJQ6q7ZRhbdECgEFfLMVEhK7q9XpRFFmD1tq9na2jwyddW7RJbZHX7Ao+YkSWSuA3qmlMFGRcrCqy8NQbrnCapuCxWESOjF3A2ytmiqwaZSykDlKm7slKbea2nqAuofkrr6Csh4XGmPl8zroLn8YbRREvlQDgnXfeMcZsbW1tbW2dnp7+9Kc//Ru/+b95/Pgx32erF29/Fyl4hcD4NYBjNnsJroGKgpJntACnjpQ6wlgaN6Eh/KE+b5sXwfRD5ejQPxTI5ric0pMxBtcUJC+HtEGQ7yxwCpfXWhsCZSCqqWn1PTLauGmtBZGjRFC3igYBHhKLotGZGgfUN4cpoMAWIXULgFIjFxflEZnNI1GSJHFk8zw34AAgy7KiKIb9XvB2qTOLH70tmHFTjlUUVRSUat+UZExNi3mzLfrnzVqtYDkZd93PrhH4CH4yVj6nHajFqywKjdpy0nWi2Ari37LtjkPIt7a2RDglSdLr9d5+++1f/OIXv/d7v/dXf/VXT548GY/Hjx49MsZwWI5V56d4ybRuLV4UXXs41hQwq+Cg/qSOFDxSAtRDMlpRGOuLsmCGBJgVfDrlUhf9SH4YXFwpCqaft9HXvHbanivrcdQmS2+gBIZ4tfvaKOe+cy6KrQb0QJERuJRIOA1AwdCAh93WRvEFNbxkcl+XmSTp8nlwAGDQAEBsEKxJEasiK4ri/Oz0/OzUselEGaZJqflW5YIQS4IkjWOklsgWUEsEvQSJbNLkFiKSTfbSb/rtuj6wCFhs90lICJNUQHpDhoBJsz3U5YG8wvp0S9LerkihLkLvUeSeYTMFeElwfn7OB/5GUTQajRDxwYMH3//+9w8ODra3t3d2dvI8Z1sQLWM5rtBsu/ZwLAwNisOozY68utdlkShPCuKAQmF5RiJbSa3QoY7aUjL52APwvjKxI8tMWGpDV4pB6qR7uHXuNaFZHvAwVJuletKit4QaY4qicFQ7ewkay2S5Y+obdkEBh4aP1ha5uktWpKOpb4lcttdGRIQuzJPnnDPWOOf4nFaJwQjC3ElFQ+tOQERrTRRFlXOsM2rF2dQj00HhrEFLDeWDfRIyQEbF8wRMLq9YMeIyTJXaCtvFohweXvnNUOLTk73moA4Lds5BRw6ZLuJWs8dlNBqxv0G2GrEwGw6H77zzzk9+8hMAmE6n1lrOZ8LdogUzq0NXZ7ZdezjmvUNOEnR5CNDTT2tAXdOSlK8DlCrRfIwfkFQ+VDdC6UWcroZMP3Eu84yqfNo5QTpjayrzlSJBE2z4fzQEdxkrSOnS+leLP1V4RlEUgLYJH0IyWHq8iIi9NKxm6ue7Ui9pzG1eUMN5u2iFwA0ujCoGEfz2mXw+AwCOl3BttmPhhMDIxg9YdXizXLu670EcYuRQfyswzW5hquf0ASXSAsW8tXOgfsp18ArNtOSVTesz8ZPyWIKfp6KFWH/Ck43b83d3EXs42Zje7/eNP4A8SZLpdJqm6enpqQSAb29v7z8+ODg46PV6Z2dnDM3SHN+Ktd7/Yunaw7EwdLAJSs/k5pRr0gIKlZUN1MwRzpPJybkLTJ0QkTP86ppAfSbIEhV9osLFrgevFsXRMoPEFSRamilCu2rQaU3hx3AMdVWOKegxgfUuW7DsfgQlJJxzeVEJLlhrrV1YBqOOae+oFlFDBOxdMIZfxM1ECdKwqhqIGFm0BtAYKvLKOwA4a0dZlufn5/3xTtADTByFFvhyAQDN0g6GiBIj2O/3nY8YkwtEtGYZiAZq9wcHJooNVzoKVXgMeRV1xbyofM4KmSBGpbsSQSt/Fmo3kEwK9MEYsl/R+Fi6deFY1Ft+NcdT9no9htqqqm7cuPHo0aM8z+/du/f48eNf+ZVfOTg44HdlWba9va0XbVcLjK8jHHMPsmGI1EEGUM9ZLEzDalGSJItRjNoHwFp7dnYWxzGnc+SFz3Q63dkdA8Dx8XGv1+MzdVxFZVm++eabiGhMtLOzU5blvXv3OP4mTdOzs7Pd3d2Dg4PRaGT8ZjDnvRxFUbD1SuI6i6Jgh6/zYUASscs6xWI/G5FBAx0rS+160ipnl01Qi6iLMCWptapM5qIobIQWIDZYWlsWObkKgazBrMioKgy42WxmEbLZNI0jcBVE6JxxjvfFACABkjW2LF2R50CGHBFgZHtESNQC+rq90gRx9LGznpyrqCJnKh+SoOewbm9kF7Db3UW1r/JyjoiAZAAXgbRE2Xxa5UWeZYg4n+VokqPjSeUMQlwVGdc2SdM8z22UlGURRZYA4jQidGghjuOSAABMYsEh78lmiw1L/TiOWcwLCS4niRc/htUFAnSAgIb4HxChIWOQiAxC5TjzH1a8QSNCCxY4oM2gXfQqObfY/dHvD4UzuQ/zPJ9Op3yYtx4UUedNfeMfk2xSZyHEzen3+4zZwvk8KZqCfDleUQTeHccGGY4h6fV6LDnm83kcx1tbW+fn50mS/PVf/+Xe3l6eUZ6dn50eG8SbN2+enpwNBoPZfNrv9aT2gA4Qoet0rJdC1w+Om0QqxF2LffkKvAJlukN6p9PpaDQiovPzc36SQXA2mz169KgoivPz86dPn1ZVRQ6Kojg8PCTCPM93dnZu37795Zdf3rv35s2bN8uySJKEIxxF68G634bqLi+ncg1r7ePSvcEXwSr7uRAql7S8LkBzapBWzfRMI7FdrOvQuVgEm351cCQreFDucvFBQ7aJeENEACehboKM8ry2mFUqEYSosbrARX38kq5SZyaQX/M5lVACVOi6ICAzm6TVR+UDFNeFsJasPPTUCHpVqierPeec9KHo7119LiSdYHzqO6hn+HzmILaSjKDwntijOQaOdww55z777LPj4+P3v/Wdg4ODN998ezgcnp2d8W7JUBG5AprytYdjDQdG7dOXLAcSkqz5vkk8DfI85xzBrJPu7OxULv/Zz37GDtksy4aD0XA42N3d/eSTz3q93tOnT/f392ez2cOHj7/5zW+++eab77//zRs3bhwdHY3HY9aFZbEm75L5IGzahK1L9EPz5nNEZMEOaNMx9WMacWQFDXU4XuKR+i8oBIhfsV79m7KBL+rbPZ69IJB8nrpMIsKF9oQMx5VPYgcKZCXXB18bvwOb6n48EdXgR4q/kpgK8NJCCtGsLh1l6vkljHfciVyX3Xfaw0H1TFuBmJQ/8zzXcf0AwGDXdKto0u4QAUrjdwyit12sHoJWCjiwOffB7xvkBfE/+kf/aDabHT096ff7x8fH4/E2b9+VyJNL1OHF0esAx6AcFAG0gdq8v9A7OspJ03Q6nTrnkiThNOdJkmxtbf3X/+8PbRxDWdo4+Zt/4wd/7+/9ve2dHSAExMnZ2Ycffvh/+j/+n58en/7d/+6/+9M//dOHjx+XZf79739/NpsNBgN2O8hpGk6Ff0DDpyfNkf17l+sKJo0grQ9fmhE1LhARr+W1KiTCRk9FVMHgARwHtdIVWwHHzxQz1Cg5eAV/dh/5uoyzFhTT9UFaKLO8YZcPdwKv4bKgdc5VrmRdcuGU899C3eSKSpUWkNV9Lt2rsRJUFC0ojK7akmWD5y6qW/Y5OIG1YLkpv136DOtfdfW/dgzqejoVpGSUz3xdaioE/CkCg3mPk5/0+/1f/OIXH3300ZtvvD2bze7de1NM8HSBk0pePl17OO4iYQXd6URkO7xkcRyfnZ3xEUez2SyKovPz8y+//DLPi7KsiqL87//7P/r9v/23s/m8yMs4SQBgNNou8uof/+N//PTp0y+++KLf7+/u3vjxj3+8vb09Ho/l0AcA4E2uTgVdCItDXXnUF+u2t3nzOWrH0KFaNmtLde2Y1Ip7ReEyWODHriuz88VJA0dXnbu6ulLpzZwOP+DjXP3vWDWuqgrdEq34jqkf8rRovgJEBo7FMg4Xz7ALDr2xW2+sb4pbfox8OApDJ3mHithMRF8u1ZkD8iurkiNqAGWS3X2g/BClSpAdkD7kTKr60lzTpBJgsV5169at3/3d3/2vP/yzvb29oiiOj4/feuutydk5WzOums/82sOx1sugoT5oyAuWewGxOjwajU5OToqiYEPEn/7pn77x7pu7N/bSfm84HgGYOOmxN3w6nQPA51/cL8vyzbff+Zu/87sfffTRv/gX/yIx9PHHH7///vvssuOYR0nsbbzjAr1dr/SngTgfgCw+k7VIq0JQVwbX7tNuEoUR6h2uq6G1Y4GJoIaa5IeIy8MjiQgXw/eM9jZrCHXZxvedq8UXQ6Ojut6iw7YAIE4TADAGLBqqm4nlU4QQAHA6JPDWA1IrBmFU5xwZRL+0ZyiX7YLkc6uDEi1awRRDsOC+rK5IWST0SgXVVkDZgR10LN+UbSna0FFVizxQTTJ+m4+WgnqFQc/y4qym4FcywXmVIy1dRNUYM5vNbty48au/+qt//ud/fu/emwDA5y7KsQxAALjgt1dOrwMcy3XgJLHqoA257tLRWIshoiiKbt68yQnYAADIPH70ZDAYJHEPAGazmbVxr9cbj+Of/vSnX3758O7du0SUZdnv/u7vnp+f/8n/8v/5+c9/3uv1EPH27dtpmiIi5zfRXhp+6UINbIuKW5eaWuozgeYSb2liaPA6rRTLQGj1EDtIl9/U0S5BQQnOB+fpzxU/l41wuiFExNpxFBljl4ZRRHRqPgdrAikBUY7hJoHmRfNxoS4IOEpwtxYJQYdjPcWV3NeOE/1DAWKZFPwTjrYEZd+XAtngpl8RDFlrt0vNwYs01kalcy5nOIZuvtVuGGl7HMfb29tEdHx8fOPGDU6Luru7y5FLKGuc5+3xvjRdLV39coR1ZU0sxfytIELTcKkpiqKyLE9PT4loNBpNp9PHjx/HcXzz9q3BaJiXxR//v/7f/9O/+bdHJ6dpr0cAX3z58P/5L//12++8s72zA2iNjR8+2v8bP/jfvv/++7PZ7OTk5PHjx5KzmFOqS9CPVKPy+Y+0piO2xXWJlF7zHLq1g1ZMRamGjAUpFan5cw3Hepp9lYY0f6JlXvC5gkTB12DECyP5qvXVWPdeuLagBaj74rSg0nAs4RDMJ1o/FSaX9Yd+r1ZptV3C+GhfOd+TF3BO7dLWfYiIHL6pK4/1bSABiVlWGFv4XCwDumMvR5rBSIl88ulfWNjwrpB/9s/+2QcffICIcRzzeeRsgWnllldL10w7Fi7nkE9jDCinfMAEzSQsiBhHSymtA2mZfXlJOJ/PP/744y+++GJnZ6ffH25t7Tx48GB/f/8Xv/jFn//5n7/91rv/8B/+wz/+4z+OouTp0+P5PP/BD34Qx/GDBw8Hg8Gv//qvcyqpOI4//PDD999/X9KqEhFHSkrQm7WWbW387XQ6ZbVa5g8ocxgAEHRaVGU+QN1H1MX0Xfe7GFQ0qUCvkZeKU0umveyvK8uSY/VBuS4XdWM8Igdg8jxP4h6jCaItKs4vs96k7ULJ5jOr4UCwI/h55SqxMklvIwIfdEI+ou7k5OTOnTvoczQbYxY7IOLU6+nO+Bwd/X6/AnTO9ft99DkZREUV1hU5QcoEIcl/2SOnuZrngmCTryrqFMbcUk75j8qsLP2jjRIy+4za08HZfCqV9hoUUKIPNM7zXKIdOHSEU0XH6XIzrejyot03qZU/SZ2jmCTJfD5nG6BzbjQacXu5IXxMOOetQY5ZRgACNAaoeuWxbtdeOya1OBKhrfkpmHXCiE11DP0qb39//6OPPnr69OloNLp15w5a++XDh3Hav3Xn3snZ+dHpyb//k//0wYcf9UdDh3Dz9p20PzifzY9PT7Oi2N7e/t73vre7u/vw4UM+q5hDhaRWUkNm1sqfgaRXtdSwPDx36tJuvmKZfKEVZFg/jWFTp375RB20+idy4epRvRpcRHlMkoS1ClF1JcOZZgDpDWEMKblV3Apiko9ERmX0YHzk01Wcc6wmi9Zv1Km15HcqdREqsxKqmDa9sCBlzwEVF6hN+c+F9LTSE81a+/f//t//oz/6o8lkAvUzB64gXXs4ZhI7QDDAWlbLHe1rkjvCrGz+f/r0aVEUu7u7N27c+t73fm1v7+bJycnJycl8nu/s7P3whz8EgLt37u1s7/IxBJPJJI5Ta2Nr7TvvvPMbv/EbX375ZVmW+/v71trz83NQwZ6g3C+6JlL5ILRIpt+L6LGA1i1HatWqhGo4fqaMCTrh1cLxpQlVIiSNTVAPdDHGsOsfvfOtqiqd94dJr+ulM42Km+ziECnWqT3Qri1/BTR6fsWY6vJ1DYWrdW31tTwgu2ZWl78uifFHSuaOOjw83N7evnv37tHR0WQymc1m5G3xV5CuqJS4OJFa06FyU4BSH4TntLjWJfCfcrLA1tbW7u7uZDIZDoelq772ja//9u/+zoMvH00mk8Fg8O3vfOfw6VGSJL1Bf2dnp3T06MFD51x/0J/OZz3K0jT97ne/+yd/8idEdHBw8O67787nczZKaCbmC1QRS8DuR29vAWV2eL5YDM8V3PXcDjQUWTU/swmoludExOWxHeCV0CX6R7TXRS43H2rmfCAN6ChAANZDva8sDOzVZWqAayocAaoG9ZcRESxmjRhU8ky2I1NdkWc4a6Su62w1+JAGXUO96hKIXJS8MlfGuqTnuLzXGHPv3r2HDx/u7d68e/cuHxEgGaWvIF17OO4izdnCMRqRNZcIP/EJ8Gma3rhx4+nTp3mel4Wbz/L3v/Wdb773/tnZ2dOnT7Ms39vbY71ma2vn4ODAOcexE/P5HCMkovF4/P7773/xxRej0agoin6/L6lYoK5QQLeqItV+EXC8lm1uBWnFSrdI+hnqCb+7ykFlzBE4JqLroiLrAWI4tv6gKafClpe2XVweSAEABEh1E4eWx/oOkzYOBFgsZgFZ7jjv2bbqmHBUiUmdc2xaFY1bDWin70He663htfxwUh+pBuscYjsW6HwuFKwmRT87Pz//2c9+tjXe4ZRvVVVNJpPhYPS83vt86dobKzRPBKv+QNfQF4J0mnsYNThZLXsqzs7OWK8Zj8ej0eju3btf+9rXDw4Ojo+Pk6Q3GIzyPD84ONjb29vZ3kOw21u7vJd6NBq9++67VVWdnp7yZj+ZAE7lH3CKwIcEsXGtyV5XU6RrFAjgGOohX6vhGOpyCF61sYI6aPVPwBsrRN8MntHjK2CqNcpgoIU3NICCih0UNoa6SNAKqZTJj0lABXtfGan1RhIdEdHVWFH5sW5wE/+N9tlqb2SXXfErkp4sunw+4dQ5xy4cTj02Gm3g+EVSYA6GNu0Y6vwKdXeWhJFXVcXu1zzP8zwvXDXe2QZrSiKMovHO9rzI415qoijt9w+ePk37/d5gBNaQQZss3OhJkuzs7HDui7Ozs9lspuFYTBDO7xuWaohcCbTj595j2EHrlhMoZU0FGepbEFfUJ5CXV1YCdVEAx8anUeVvZXDZeMq2KVfPLNFUdaUb5Um5NupcleBXoCKOQeUXBrXp2fgcpBq8gmmygh+c9xCC176bc01fS/nadnwJX8WKzteNFfm0s7Pzm7/5m7/1W7/FEohlIftyriBdezhuaseBFhYAdMBhgkFZlonfWXiU0bkoiizL3n777d3dXT6zfTTcGgwGfK74t7/93eFwaK3t9/vsitnZ2eGsUScnJ1tbWycnJxLaaerB/+TTccnECJDxxeHR84JjaMxerUVKn18cW1+E7HnJJEprU5oKo3IAnOi2gtS6HA2OmmekWNcd/myX6UZBoj+d3zzN0W9xHPf7fT5zWkdTBBk1u9qIDYeNXOvJ6F5KZIXmNDGPENGTJ0+qqrp9+zbXpCxLTtz4vN77fOmawbGMn5z4CX5XcRzHsp8CfTgk1hdr/ICcGgkAkiyccxCfnp5y9p+9vb0oih49enT44FFU0Zs3b4+T9K//8qf/9//xf9zqD12e3b15A4m+++1v37qxWxVZUWSj0ajf7zubkE0GW7tkE5sO5iUNtraPTs7y0s3zUuKOOXWcMaZwVQVkk7hwFVjjcJnR1RpTFgUQGURyzhpjurlIQ+FF0Nx10LojojOfkc/hwm3k3HicjYxT4gEAUARglv8Igbi2rqoqgqpyhbEQxeBcaW2Y4F8vdZ+XOGmlrvItRlQBOAtgwKEBG6GBCoo8d66ispzOJtYAuTKyiKYyBsoyd64sisz5s6l407MxJoqSsnR57hBsZBODC6zUgMWNiqIoyzI59CDP87Is+cARowItRKhzIB1vbhJVwxjD6VOY2/M8F7ee5FM2xvT7fY7Vla4WvuLS2FmCPlpfVG9UEXXgFVV+kS5NsJIfYNkgnk/jQ/K1QQO9Ok8dJIwn2vd8Pq+qKk4Gs3k5z4usKPOyiBI7y6ZRaioo4jTqDRJH/hhatK8cD6+9K68pZlfPSVlVNa1anF7g/Pz8yZMnfL7LZ599dnh4eHx8fOfOnf39/eFwOB6P2Qpx8+bNihbQyX48YFU6z/u93nw+Z9a/cePGwcHB1955d6EmNKK4miAitbpe1NqQ5opEHmtiaPDwlVVhWgkRAZBUq/UgBs2UhusVPd/RycaMymuh+01/in5qVX5tIprNZlKI7v8gBYfz0ZaqFS3uWT1Y3C427gU3pf6gAvzB2wDlWpYCRGSN1a+WLnIqclm/qKv/bT1FOKmESkalw19jRF8FXXs41pm35OaKftfraM1kVVVNp1P+ajqd7uzsfOtb35rP55988sndu3dv3759fn7++PHjb379GycnJ2+++Wae571Bvyyr09NT8DnIZ7PzYZqWZdnr9Q4PD1nHESthVVUGarzbZDLRAp5/T70YoobHH+quPFSuKj17F/3QBhnQcOtdfTLGOEegwiFkPxsq0asRKugrj79VUKz8VthG7qO3AEA9w2QA1lAPrg/WT1on1a/GhrFFJAT6lBp63hmVc8Mo16Kpex2F+NVGPQPeocIFaji+CDVnvayJZeWn0fkK0vXTwgLSViq5uQKO9VJXfl5VFS++eKXmnDs9PTXGvPXWWx9//OHh4ZMPP/zwzp07nLj62995f3d39/j42MAiOQBv96yqKo1jXh4OBoPPP/98e3v77OzsnXfekQPEVvheoKFrXAsK9Bd9HxeBw8uDkTQWB9fQYWnpWpy+mtZ2k+YrOUyzqSY7FWMj3aXhWOyegiByRy70NVtjxdHHFdAr/dYek3cFoiK4aDZNas7mPqNcebwBNUhwoRHZ+C1/Qf4WqcCKaftMErkizlJJ6MExJOIev8qT6+rW7AVRK3CztOeDaQHAGHN8fMxOgLt37vzXH/7w8eOHfAaXc26Q9u7evcvpgSKz9BsgIptKt7e3v/jii/39/du3b/f7/eFwyCZjmRia85rLsa/ClC+fmmtJ3SKBYz3tg+ebrW4iwtUnPaDiKAsEFfqYML08gnozxe3GaAL1uF3djas7x/ld1xLmEccxW9UEsvV1Kxw3Ba0guHi80as1lzs2oTkRoD4XAlpRDta1ASlQjgZGv7C4guKc6drDcaDtMq3obrH0i9xmcT2ZTFj5JaJ+v8+njs7n83v37iHiv/uf/+cHX97/8ssvHz16lKZpPp+laVoUxWw2c2UBsNB8y7LkCOUf/vCHs9ksTdOdnZ3Hjx+jUlgC5Go6begrpB985YR1PbfyJwxdZDrJr/hOU0pdZFq+KtINl1NgtGODq01+L778KtBSjTrBSApfwQ+sbxofz84V4EA6qxLMiubYtAth3bbQHAjdNP2ALgG91hzEUAtYk7cXi6NS14dUMAZeanUYRCgJ8UDoDEdXGY6vve24K3Cya8Y2VWO+YCzmoB85zjbP88HW7mQyOT09/ZM/+ZNPPvnkd373d3lu7O1s88Sz1iKgMWBMbK2N4/jHP/7xL37xi16vNxgMEHEymezt7C7mGLhgmlVQ28ktM/ZF9dfzpkCjkZukIr3EFI4tGb9a9v6iSo78zPdeKeK2yBGigaBFrx1Dw0ChPVeo1hMs0viscc3hco0+75UgjvGRCQw9GgoBQAIYSBn3m3AswkO6WQ+THlz+VjBOb35RhZCWDUHbRWBIycYYFmn64XXFsPz2GtmOrz0ca+34mVgM3uOhOZuHimMqAGAwGEwmEyIaj8fOudzRr/3arz148OC//Jf/4pzb2929eXPvo48+uXXrFgcbcaROVZRJP7Zxcnh4+G/+zb/Jsuzu3bu3bt2aTqc7O1siq7GxBq/8KT7QUEOuCz1T+AUTSbcu+G3w1fOv6wsjLWacijtenuOkntQaqAYa8okiRWcEAOccnyyjXyTXlT8Wr/LnijLos31MDB2ifoLXVbkEp/JpQIc3TDctUJmbVRLsE5VZ5peeqlIr/jOAS5HH2q1n1AnxTZKfa/Ox8wegOBX7fJXp2sBxILqbQwtK8htjxPQWxMND5WTTjlj62Sjc6/VYE4njeDwe53m+s7MznedFNk/T+A/+4A9ms9k//+f/PEmSf/g//A+IeOPGjTzPj0/PxuPx7u7u559//sMf/vB/+lf/j3t377733ns7OzscGco1X0RZGEDEPM8FoCNcHmyqzz2TKZrn+XA45CREzrmuA+RWs2mTnhdrkopJEnMkeUsFEXHeFjGDgkpDjogSimD8yXLWJ4OO486kt6+QaGmEcWmcxHFc5nMiqlxV+eNK+cnxeBxFUV4WlT8LwxiTZVkURXme37x1u9/vAwBrgmgtsX2DSs6YLJojIsZxLCZgLpzfxU4qUKDJlhB+kmsiZ+7Jz40PO2PO5/GS03CYUXn6GGOIgBeLzJCSnlt2iLCNzqn8RMaHBpHfQMisO5/Pe70ep34n7+nl58EHonD8nFRGKgzPUlO4iyqV/J6IOJh6OBx+8sknANDv93UikdFoxOU5B1fEvXdt4LiLnNqNivXjl4L1L19IPJCGcuf3KYEyhC0C7NEy8kZRNBgN/87f+TsffPDBv/5X/+o//sf/+K1vffu9996z1n7x8MGHf/3RJ598cnBw8Hv/u9/ZGo+3trZYawYA7UpW+1S8RHHhjlKpcPDn4mZHP7xyXVJq2LxvVFRWq3r1OpFeX5Mi/YCgqlbZ5Bk99MIqgRLd7G2j9mRTYxEGau0lKjN470Xljw1rNie4KaaPoiiCxZxMwKBdTDwFbD2tXVmWiX0GBOmWrmByNm6ISbry6Ti4qqxjyc5DDn8CAER41fOmRtcejjX+BtzfCscC2aaeKoU3j6H3vQoi7+yNTk9Pt7a2+v3+YDD4/q//2ve+9704jv/yL//ygw8++JM/+ZPT09M0Td96662/+du/9fZb71TF3Hh1w3mPNu+PMsZYs2SvxaxoZDDUEw/a9rlcKQpqC/WZo9V8PQQBKF/Z1q1FRIQqkzXWT3gilYXDNPxpAYgHXWT9mbat/SzrRSkKVJcGyJ79/9v70yfJkus+FDzud4s1IyL3zFq7qnpBN4BuEATQz7iIBEUNl2dGzYy9DzLNmCibN7Ixzdj7Ov+GZHoyDYczZmOmL5Q+0EQNOSAJAgJIiFgaAAF0dTd676quLbNyjT3u4j4fTvjJc/0uuVRWVWYhjrVVR9644dev+/HfOX42n0xoE4MXUV3g7oqS2SS8owa5C87iBGBLEtsn6wHBYtFgFi3ekvG3LNHUpTiOcXNJcii3q2eBzj0cW4yrC8xbFnfmSnLi+4QdBymkbnfmmo1WrVZzHGdvZ7ezMP8//U//+3/6T//pYDB4uLUThuHS0tLiYqfbHXzyySc7O1tRGNLp0aQaEwPk6iyQ8m4JwTKy4Gj+h7OgHXPk5T13zClZ1mQ9S1jMxWeSKRdFs0NgzS02QgjFNGhrQCRLKsuCLKQV3qI9Fn1Al6Bmzj2OzrxLVgvA4FhnqvRpU4mFENb6FRgvn04f23rC4S4gZDY8rQrfDisHodUiMUf2nUEjGNKzAMecs8nkpDIHHwAzZpGNAswUUogM12K01oNub3V1Nah4WmvUkUeDYZQo3/fb7bl2e244QjvX9BiuTqczGg5pN4fxRtg9WinaGM4gA8f0XrQIuV9ICPG0UdcmC4WBDTX+S6NKyl3q/jOppJyYCOZQtJNcx29x0ulwUmu4LOWAQzDxswWOvHEuv3nLKh3dUUT0UH6b1lpK+wQmYJMu0rq89QhiYNoj0lMSc/RJkbHC2lLkdo8Tmpslq/k1Ho/DMGw2m57nYf0KOiaNbJVnzVhxNizYj0CcFThBRshzLpfpwDhh4oJ5O7i0KpVKr9cbdHvC1A9wXdeVIkmSySRKEu15jta63x9qrVdXV8kNggCE5Yqs8x850UVgCgXJFWD7u0O141w61cHOoRyEZdsUxQpIigxBZgGfa6L5Ir9lFgoTc4itxZnZWdMsFiILc9rsxyWLwQDG3pABevxzMpmgmRUhkioQ8UeITNKgZrYIYU7UTtgZS6TTWGjOu2o1BcX+5+xvSSsvuj87ntjDWq0GAP1+H29w2BGuhgPxcdP/ni6dezjWzLPPP0MBHPOYCsFceYIVg+fsiAFwo9EIEtXtdu/cubO3t+d5Xr1e1VqPRiOtlOtOowLCMNze3u52uxg7gRoB5T5ZDhnqGL0IMGOrYBo6MJh7YgN7dOKDDJk3EmlXXvaH1oCcd9LpAqpThy3bnFlwTL/i0tdCMQsZ+Q258bwOO0PdgmM8tJQ3iEXaLJ60nggMoyWrrKYYWeKf4682ARv4WRoqqc3C1zIfk6L7rapy9O5BEEwmk16vh6f8oCCxWM509enD8bk3Vhw3skKyIETaR5OfAdh04vSPRqNKpTIajT766KP6XHO+s7i/v9/r9VqdtpSu1nowGIRRNB6HvV6v1+sNh0MB4DgO8r3jOFonGDdqwc60wwXvldVNSnjx7BBHZJHZyZa873knPl/ESFqnzMTASqwUwbH1gf9p2kwBU5E4p28tQBcs/Rr5n0IOinhMM+WGdBruP6cNgRAC49iyhJo4WdLRpAAAXrGxgv61hiiXyEYEJlAKAyqklOPxeDgc+r6PdWmUUpi/fgaVgHMDxyLteSPp57qeSO+tyFLJLXf0OdYKBAgpNUACWmkFAODIJEmkOzXzR3TIue+pKMYATCHEeDB8MLqDILv5YBosEYbhYDBAFke39XTjJrWEBAtWgSNAxUJKnQkbdgVoAAFag3ZASwEOaFdANB45oIVKAtcRKvE8LwItVAKQr1A8Ld4KpQsADggQQjuedHwAoWMllFSRVnGi4iTRSgon0Upo5RktZgpbYrpfkaz0rZRyMplIxyPz61kiBSBwn6N0LB3wfTdJoiSOXdedjAZa68Gg57oyiiaVSsX3K0rhudEV13W1PhBRvu8LgUYDoaXrehJAY+EhxdIoSF0gPErt8YUXJ4lSwvd9jL2N4ziKleP4SmsA8ANfmlprcaIDXwpmCgOjfyA4gsFKan4SjpCl4ySEZArESThN8NNagwA/cJWSGJvE6/WQ5o5YieENmM+ijYOH3kWZ+GVeYsJhUer4K1AFiKx0rVIdjUaO4zhCShDheOI5brs1/95777XmOqPhxJGe71WkA5PJxPM8xxHmKVOrhdaF6+vJ0LmB4xOQZtY3vmfMJcuIQeHJkkE/egPwflwteBuFvqMljhPXSuhfS8gLRpBWLelFTnloHg/x/kNmmwx5MsP6iXX9LL+41hrA1nDBvDW5iyHttyCl0iq6pjJHgeRyRZaSKHSlUCBUHI3CCbZc8b0kSZTGipoxGUqx4pVmx3YQ+Ba1n+0Acb7InJKHAlUwGyDkWaL5T4454IVE+wa8Eyvrozkbz3loNpsoCQQzGx796U+Gnlk4xp0R7V+kqcxCeGqRZBWyyd6PzMark3B7FkX8YAsY1gYsbJkWm7WFROKwK1jtgiz3W6B2pog2ffQvBxF6NZFHua3BWX1TJN5z/u40j4k5t5TKivIbyP9msQd9xccnG4uSpck49H0fT2UkzQMAKPPTYSdoAAM1MjhYbJx9X8myliU7eSSXKJiM3shhxbOAwbFIm3FKxplIay0K0lIFO6oqjuPRaDSZTPBYk36/DwDtdhtTZIWwK3mdHXpm4dgqQZJrsOPEgYBbRZIkla1Hfjk8bwarDqEhDB+BcZ2c+QiOIQM0WQjLojCw4IqSZXAWyMIRrh3n6iPWi+uMQn0GteM0h6SUR7yIZituKuXMwJOVOUTSPcoEYlqQXcS6vitdCVJorRMJGgDiMJwWrHBdKV1hggZwAjD+kjQG6kARa9HEcX+slBLzpIUp1ImmBhwTHg3Ch4uvCC45isaZM8zBWi7W4skwjaUWoyhaXFyUUu7v7ydJ0mg00LpCr5Ad0qeOzs8sHFtp0MQERRxAARWaVY2QUnqVCt2QpOvPEs5SIrwFH1zrgbwIBCGE1gcrljMft+6dfZ0RiWMxwbEo1o75iiU4pnb0mYRjIq01wMHGhYwAmICAcIwRCwfyxrAW7eW1iVRznClH8d2bxTBF3VDpYz7wOhkTaBbQ5xFlskCRiuDYEjbiMJcyrSCZPtGZWhNMTp8iEcQnphA+AHQ6ndFo1Ov1giDAsAqSeVadpnJd7YnRMwvHwkTkEOuTLSKXSEOxFBY8y4v/EK/XajXUQfAKlWLJJptm/+XwqswhBbwnFnPQuj17fi0AfJ00yBKjkxJkqSRZyURNscF5+ssjS5xDUpuejHaMpYLSovcAjjmzSePjpcA14pNDMcJ1hO+7juPEsRyPx5PJGOVBONHoN3Nd13HREKFUojAEXrMtY3k8b+FzXVfk2Y5VYsfGcaWEXg0Ok7VcYENKO86/n1pDLMbwiYWFhfsPNsfjcbvdpuN+hAmHP5bl+snQMwvHiSnFTZyNrFOUJk/gqJlVSyk1DiMu0gllpDn2nLBSa41nJ4u04Tir4/APWYWafpXVjoUQcMb0RQsyLGXQumJJpqygKm/8LBDxCVjzYvqZtR3zASEGw4PPrWbBFGADw7EWcGfJ9/0wDIfDYb/f39/f73a74/FYm0J6WGilVqtRuLFf8zgWWxCZJf6mcIT9Cu0ySS8WpiIzsLIScFjgWrZL5fdTLArGlgBApVLpdDpv/PDHSZJ0Op1Go0FGbSg1xz9FembhmCfCAYAwRq6iGaUAHTBMgwq1a5x4kHYK426I2qSfW7Y2SHMwYTRd4ayZVbuyLZwvOpT1c7HYANAT6OBpEp++XOWLZhlluZM5pl6ZIqsEglyfyH3o9vb2gwcPbt++vb29jZVSpDmHCbmxWq3Oz88vLS11Oh08B1KnDWjcU5elhJ2jASyAIUmSXNuxUpofRIJvRCGMkJ7rQxXkow49k9y4OxFCuK5bq9W2t7cBoNls4tFrQgiAQyTQU6Qzp64fl2j/zn3WYJQUxQpR4+est1qbim467fJGNx0uDyzWJ0z5K9qMIzfHcYyO7Fqtlpii14k5qQxvI4+fZKfvaFNtGe8huyGqTpY/R5j9fi4Vjc9x7z8ucT0IgQAXJ/o2cdHiC0pWeIFG3oSvKKo5QFNDoQJnitBFTDNCJtowDF3XHQ6HQogoitrtNr47ury0ceJRYSkp5Xg8BnP0MjZOt9HjlMloCMMQy/XiOeWo+d66deutt9764Q9/+OMf/7jb7eLOrNPp1Gq1arUaRdHu7i7y4bvvvnvz5s3hcDgejweDAZ4iNhqNlpeXu92u1hqN3XiiAj4UjWNYNl6bUlCCWX6x59il8XiszAEonEg7JpYG467EmyWLwcCxwvNPVeYEUq50J+y8YDBGMFyD4/G41+vt7e29/PLL+/v7e3t79Xr90qVL+JXv+ybo2OHWdr7peYr0zGrHxOJcDPJ9X9FG+Ih7Z66zcHzhgpc3glCL2K1NQaw4joVIBUsht/FtbPke7awRR9uib60r1ofcb88d0SYJ2AuSTqqZ+Xh622FGA9/39/f38cQvxJ2PPvro/fff/+H3fzA/P//Ci595/fXXX3jhhaWlpcXFxTAMu93u1tbWG2+88cYbb2zv7K2srIwn0Te++a1XPv+5ZrN57dq17e1t3/d3dnYajQbKTtIAiOUQHJWpx4YZ/wjTyM+axYqgFsxt3/RBsWQ8Uo/4YuQrqGhhIlnlDWhUscI9YvRwOPQ8r9Fo7O7uov2w2WxGUTQ3N4dYbCWFnx16ZuFYGseXY50GYqaQ1Fsojj8DAMizG9AqUqzABXES5y36YWJOk6Rq31xT4G1KdkI7f+7ZZCBO1iLk1/mwWLdZNxMWnP335US6P3b+YAPEeIZbCXg0Lg1Yya6lXq/T3i5Jko2Nje985zu9Xu+LX/rKb/zGb3zuc59bXFqajMdCCOkFFS9wvGB57cK1Gy88/+JnvvOd79y+fdtxnMZc+7vf/e5XvvKVt99++4UXXkiSZDAYdDqdzc1NjMmVLNgAuFABgPRuBsxJDpRKh2DNjYQEuOSzIZsM7UT5kiErTcnUK5ZaAmn7D+q/SZKMRqNms9lsNn/2s58BQLPZbLfbg8GgXq/v7e015+rD4ZDvO88OPctwjMYsaTJEwbAXaSiQF8djYUSu/guM1fhnrvUAg2O6Lo0DEH/lum6SaKsPWYshtXPWuAeob/jKLMKCbqC34yNDN2RHOzs754LEQQXUKdpOX5YVEqL9O22PxIEsn8r1ov3y/v4+JjXg/W+99dY777xz9erVf/l/+p+vXLmM4+R4vuvKzc2tBw8e1Ov1jY2NJElWVlb+4H/7v/vbv/3bv/u7v9vZ2++0mx9++OHKysrm5maz2azX69vb26g2ct6WrNQOlRwimx4XmcTVqOuglmqtCww1kaxmkGJxpSKt0ABbC0WU3TLiqVfaBLe0Wi3HcT799FPP89bW1nzfHw6HtEJ5SqQ1icee+FOlZxaOrZXPMRRvoMWTix30r0qXyOCrTrBUJZHWlK3FCcZViMytWElcrFrA8VenK3NbrH/WIis4ERbLdDlQfp3/aU0Kf2X64ZN+h0cgwgKOtsJMrpMu7ehkDkPSpfaKIAh830fL78bGxo9+9KPV1dVf+ZVfWVxaAgFKQxTFvu9qgNF43O31ev1+vV6fTCYbm5ta6/1uN06SCxcu7G5vSCnff//9+fl5DMUNgmA8HgdBwIORyKqGEWPYedzYIfhavhbUlJVSUrgWmNKwIKxLFlmRJIl0vdz3LSLJMgOAme9xx4l1Y6SUqA6jWRxPEEarMbpwsh6U8sF/YnTuXXlFRONLHCaYMsslPFEufFvfUguKlUwF5k+3sIYTKhfKZKaSBS27b7J0w/MCT0XvLtKUvQ4Focdn/H0twp7TtOJFBAvJjokjycRhy9IoszSZTIbDYRAEtVrtgw8+iOP4xo0bQoh//+//w1/8xV/duXN/a2tHKdjfH+zv9wDkcDiOY+U4nuv6nhckid7b6+7tdefm5nq93osvvnjz5k3sEhYtIr2VfIlclyRZgjxM7kfNbA7kLee6BdkWuEedbqZ2VDoquUQ15i3zQUN5EIYhJn0sLCxsbm7izZ1Op9/vY30ljAvUxSdSP116ZrVjGnFtYtQhY4HNwlwWi7NcQgzksINeFMuw4hBDj6BHI7+iphDHsetKLjDIlWdhcfne7SyQEAchRCIjxqxhyb0HiUnK84TFYMyaBMec8cgfAOz1rS1R+RQHQYDaX7fbvXnz5vPPP7+0tPTw4cP+MPrBD3/0/ocfbW5u/tZv/Var1QIA6Xora539/X0waLu0svqFL/7y9evX15dbQogf/OAHnue98847S0tLL774IrIixTmg/ggAmNwEzIEmzDmQQRAgININ+JpJfBC8xLcCJH44jNK2kmvlBM2545Ckz1kXzJWHx/H0+/1Op7O4uPjOO+84jtNutxcWFkajkTbJAbo0O/fp0lns06kQl9v88xF/Xq6X0VrioElXcn8rhMBMLewDlkPUxtfMHRSkjEAGhct79dSJa3NF18tf4Xy9LyedJsJZgmO+AeICnsRtOXPiRnsymezs7Ozs7Lz22muu625vb/d6PQDodDrINmghdRxnPB4LIdrtNhoWXnjhhd/+7d/+4he/+MUvfvGVV17Z2tqq1Wpvv/02nqjg+z7CMUdMtGNgcfrxeIwhepVKhbvsyLNH2YaWck0aDDI/gaAQAiMgrXEgxC8aB+4L5dkAAIBhFXEct9vtVqu1vb2ttV5ZWel0Op7nYeiedTLWWaNzA8fcpQsGuchfByYUkZAXYQ6MBKboxXA0TsJIKC01CKVVFCdhFE9CqQGvCKUhUTpO8D86BBpNZvgZOZ44WJnarLiPw9RYx9TqRdMbiWVhystilKWUvpQ+gBTCkdKNoiQMYyFEtVqdHm4tQYMajYdCgpC2+ZsWz9OYEwAAqV1HuRIcnSgVxQKUJ6WGOKg4GqIoCV3fSbQQjhdpSIQEHGjz3/SlJLiuC0nsgIYkFirxpCOS2H16lnJdQBKEihNQ2gWBfCKlRFgRQozDMI7j0WTcbM0p0FpMa6eAMZViqCwFhCEnoBsNmSpwPVdIT04/OCBAyyhMPDcYjibV2tz9B1ura5cuXLxaqTZB+s25+Vq99WBjO4phe6dbb7T7g4nSThQDCC9ORJyI8SSpN9rPXXtBSP/eBgzD5v/l//Z/397tvfzZz/z0Zz/o9zY8GdV8t+J4OgQPfBd8HQnQMooiKVzX8T03kMLVSqgEQEtHeuNRGEdKgCPA0Uo40vPcQCtB68V13SAIKpVKEARYTUmzzaXWGhcLgAJQQugkiXzfBVA4Tng9+5/rSteVGACNP8T/tE5cV+7t7VQqfpJEk8loa2tzaWnhhRevDYb71ZonZKJ0qCESQruuTJKIpywYdf5psduUzpOxIis5dcZcS39a0pu0TgVxbuM8bgYghXol9xdRrnin7RUcQe/jmhQcQXs6O2QNHd9X5t5JL5ht6ry8MicOOtYkQprBdNqlQTeQlop4ofQ0NsP1PIziunHjxoULF3zfb3c6m9t9TPdYWlrCJDQhBAryarWKEcorKyuDweDu3bsrKyvjoVRKVSqVa9euRVF/YWFhY2NjcX5prjmvjYGI9pRJkhQhlMPOP0RFB/Vi0li18eCh7mxlyRJxbZrMyuXDm7ufwHDjnZ0dKeXrr7/+l3/5l8vLy71eb2FhYTAYtFotrXUYhpVKhVdktuipq8znBo6zWxhtystSBhGYTCFlkpUpg1ObPP2iES8KBihChCI45nZD3oKFSmx9Fr6v9e5nFpuwa7nDqjPhpZBnO0aycPmpr42TkWLlg6035VgsWbYFgL3pIYeEEEKDBIDhcFgVtThWW1tbN27cwDSHxaWl1QtqOBzeu3ev1Wqtrq7u7u5SWgdG2tZqNc/zFhcXkyTZ3NzsdeMwHNbrXqVSefPNH66tLvT7fbLYag2O9EBqrXWs4yRRWuUsAeq/TId1WmCqjxAsn7CqWNysXDK8/E+C5iAIwjD0ff/evXt/9Ed/FMfxV77yle9///vtdvvu3bsYzjQajXBwss7zM0LnCY75nzQNwNgXWFaFYsWxhLH3K6VkwVzrvEiXkyEgcSf3H2ax2Nx2yPuece1Yaw0ayO3GxRj2HCUiikxrDWS1Y/osBLlhz9yaKSeKcNfMlyCMa4v8VDz8nAtskYn8wWrpeEOlUtF6eoBTpVKpVquTOJybm6tUKpjT32w2XdftdruLi4uU3xzH8d7e3t7e3ubm5tbDXrNRa7fXv/jFL7777k+r1WoSTbrdbiVoKKUBHJBCShGpRCVKQyLFNH2DXhA7QwewktUFJxfrKXO5a4WUWUSaEzAecIqPNIX00iD4xkRErfWDBw++/vWv/9t/+2/39vbW19exCD35z8l25OTVb3rqdG7gGDLaE34ITOKmNr5amnvN3LjCaMpJGOU2TgYNQvOEVQrPUgk4ChOPzMDl2PPNnR6QguOzwjpIWmsBQmtN3eKrEfVENKPTqstqx0IIre1QQlHq0jlrRF3l+Z/cksYRFkxKG+eN6c1pkSalRFBHZ1QQVOfm5uI4bjQaUspJGC4sdvb29hrN9s7Ozn53FASBdGBxaV5KWF1b3t7eHo8nrUbrJz/5yZtvvnnx4sUXn/9sperWq25jruX7/mg0WuzMG+2YM1uiIVFau85BTQkwerFg2zVLUbAkEN1fNG7Z+8s1jyzz0Frb399fXl7+lV/5lUaj8cILL/zH//gff+/3fm9ra+vChQvD4RAAarUaZosIltsCeTalp0XnCY458TnLfhZCkBikIVaHHUBwWiRYegg9V2dSSMrhlYMRaQFPnV2yZIY+J1IQiUJui1RjLrT4apyOwNl75XKy0vE5HhGuCRMny+eUy11rGFHDHYeTJNHNZnNvbw9rs43G48lguLezu7S05Lue73qoJrdarTAMP711u9Fo3P30zv/z//FHa2trv/97v1epVFTiua6YjPerQb0aVKJ4lCTJoD9qtzSAg6Mda5UkiQaQTqHFHwM2BMtsxn4SzJH4KbcFcxlM8ctQrO4oVlSAHoohzEmSvPbaazdu3Pgv/+W/vPfee8vLy41GY2tnu1qt9no97PN4PEYrdjYN5yzQeYVjouwuGP/EgwspEIJcCkFBFhAPdAfDc05xQc6iudQsyInfUyzwy+CYb9vPBRxDWtfQJo8AWIa6pd2Yn+SkR5470qzAfBHU4hULjkkyJUmsWIaR1jpRCTpIhCOjKFpYWMAAte3tbQ3w8a17vu/X63WsLoQRRA8ePJBS9nq9//yf//Pm5ubv//7vf+YznxkMBgAghBIaoih0XVc6MF9vj8fjarXK50UpFatECO2wyH0uI7mOTNHHSByac9XnLClWwoI6UMQACavfD6acplLq3r17Ozs777///mAwuH//fpIkr7766mQyabVaqBEDAMbq8Z+fNTrHcGwk6oGBGK+T9BbmHEliIKf4oHg8/JHC43CLzQPpLCpx5QFDT0tzpx4eiq3Zb8t5+umS1hoECDHN4c5qwfTZ+mEeLp/pNy0h3nkLZPEGrviLjCmG8EWxeg4AoJROkmQ4HHYW5pNENxqN7e3t27dvv//++535+Rs3XlpZWRkOh/v7+/jDarX64MGD0Wj0V3/1V+12+1/9q3+1trbW7/drtZoQwnWE44haNajVK0mSNJq1Bw8eoDFaCBDCUWJq45auEFLqROt0MCVHYcUyOHDFDQYDemV6/ZItqWYZz5KlaRSZj3l6NBhzPIqr4XD4J3/yJ1tbW3/4h39IEdDLiyv9fh9X92QywVzwctv0U6TzB8fWVkWkCaeTtlHA4AAn3p26Fg7quSC7YMFAwQ4nhbwzmejRlN9pLTlgzhxya3ANwupzHE/j8Eg9GY1GmHqP7mAhXGxwPB43Go3yYcnS4w5JllIKPX0Q6lMY3o/9GY/HGNmCoxqGYRAEkBNeomlDg9PkB9PAmEQ9HWguEpaOdKYzpTWec4hiHrNvcaaSJEGjAfDSbibSlnb0mLaLUepYHB3VN5G2TQkBrus2Go3hcFirNZaXl3/913/9z//8z9fW1irV6vLy4sOHG0mSNJuNJEkQpre3t19//fV/8S/+j+vr61LK8XjkujJJEtd1onFYbzSTRH70wQf9fjeKWq7jjMfjIKiBIxwhPM8FqbVQWsQgpshYrVYBAItt4nuNRiPceipTXROXHhWvgLTpnJgfeVsbV6SCA101iiJgYil3/ClSTRhlK0mS8Xjc7/fjOL579+5Xv/rVF1988Qc/+MHnPve5hw8faq0RgjHYDqcAX+cM0vmD4yNSITwl9rE6Mn0aAseIojUJaf0XClYvBx2VrrzOPkxPKKCdHRctvH3jTik0khxpXB4PaebKg4xGbEkyYJZ0gMLRO5t0oOFmrgOLGi73AGdv4AxpzT5dCcNQiJEQ4vLly5988kkURdVq9Y3vf/9zn/tcEASffPLJrVu3dnZ25ufnv/obv9FoNMbjcRyGKOSajUYURePxuN1q7u5u+b68c+eOiuLl5eV33vq5UoiKUms9Go20UF7gA8gwiaQWZMFL2LmrxP/UW74R5NPNJzfLBjSOnNvLx98xVfCFUaT29vbu3bt3+/btOI5/67d+680333zttdd2d3cXFhayppKS2XnqdEa79egkCgiMvUkbjwqX4VYjJcxhGayzG7rsB34z37daPxeZ+jL4Wzre5iyTYIFNJO14lIguoKfa62OQ1Vv6TNeVqQUM6Unn90tTwoI2+7moIVhaEwZsPXz4EACCILh27RqejzccDsMw/PGPf/yNb3wjCILf//3ff/3116WUg8HAcZx6vY6Jzv1+fzAYjEaj/f29TqdVqwaf3rlVrQV37tzpdDqO4yF3hUk8Go3CMHQcIT13moeiVGyIhwZbooVQO0sWIlsLx8Lx3JVINJlMcBdCVovJZLK/v+84ztbW1quvvrqysvLpp59WKhUqG6TSCYGCRRmeNXpm4biIeJgOCXbuvsPbDoUJh5E0tYw5M1kf8Db+E+t+C6OlCbMntZrOZcilpwVzKU3HXOExbZJV0xfMmJ7991xQ0djiFWUOAKMAA5pcSEdWSOYly6IP5xy82fO8VqullBqPx/v7+6urq1EUvfXWWx9//PG9e/cWFhZ+53d+58tf/nIYhtvb2wg3yGN0aGmtVut0Oq4nb9/6GEB/779/p9PpfPj+B1/4whfQrjo9FUlrNS2lEoNQ1E9L1bAWkUhb5CyCjF5iRcvlvn4uYRI2/haBeGtr6+HDh3ji1O/+7u++9dZbV69effjw4fr6OhrKeJE5MAbJY077E6JnFo5VAXEA1awqYC4HnAwmSriQN1sOl2RNJo7HwoAn6M+TJJGuFyrY/gPOlVGiiCwXFl4UTOtHXsJoKo7FHH2cvCONebN8ovEG13X7/X69Xt/Z2en1er1eb25ubm9v7+rVK5cuXbx27bmlpcUkiWu16tLSYhhO5uaaSRLv7e2OxyPf93zf01qNRsP93Z1f/uVf/qM/+qMkST772c/Ozc1tb2+r6emOWoCsVqu+74dJnCSJ63t8+yhZ4XZL4hYNV3beBdswZRcCXya5REnYaDLGw7NHo9GPfvQjrBz09ttvr62tRVHk+/5gMKDx5HNxZo0VZ1Rpf3QqQS6um3CFlP/wUOCjfRndyWeacyEtttyOCeFQr3gfCI6VOrCRKaWgQK4XdfhxI6AQAgQApJaTBcdOurYk79i5g2muG/Ne02V0NKF+iu/Lp4aCuiDPcJEkCTCg58Pied7u/p6UslqtYuVMTAb52U9+ury49OKLLzpCDgYDz/N83+/u7at4WqhIgogmoVKqUqkEng9z9W9965s3f/aT//n//If/+U/+5KWXPoMlflzXBwDHcbzAT3Q0SUYKtO95IrGVXHwjdItxSQMZMLWmmMO3MrUEdMZJU47IaDBBLB4MBsPhcDgc9nq9yWTy5S9/+d69e/Pz8wBw5cqVOI49z4P0PkOly5SfNTqjUuLRSRQQFYAn4ltLTuXaKzeiQcZmCkcGGuJjrjIQswqmHZ9x2zH1zbLDWEb2onl5eh0/HnGu4NNBcKxMeb+S33KMpkKRtK3mRD8MgqBer9fr9VarJaUcjUZ4SrSU8hvf+Maf/umf9nq9xcXFbrfb6/WuXbuWJAniLJaRE6Z63N/93bf/1//13/3Lf/kvf/rTn7788stxHN2/f1+YMDUyEONzOZPTTKFmSuep452CldjnlLsKRFofyg5UCT9gEAsAdLvdnZ2dvb29+/fv//znP19eXsbC/JVKBauMou2F8xhNUFHjT52eWTiWBcSLC9OdpMvglaOgnjA6ILcI56b6ICtwkzEnvkQ504iMl5l8F2eKsisN8iIrrJtFWmd5ct19ZMrur7NfAdskWW/H1V4uiQU72JQa558Hg0EURa1WazgcXrhwwXXdWq0Wx3G3261UKj//+c//w3/4D3/xF3/heR4ehDE3N6e13t3d7Xa7GHz205/+9I//+I9//vOf/+t//a/feOMNNHcAQL1exwKhWusoitCVh53BUkSWK0ybuGCZzuMvErFZDsmOCVdryvlhMplgx1DwoL3i008/fe2115RSc3Nz9Xr90qVLw+Gw2WxmXYuWnDtrdG6MFVytEOw4SB0nQkphWEUaYyXGMGYJRSufEgy0yLIUpLUYSK83PB6cIB4fOplMyKWu0+Yq/nNsX04LMaPEFkKAlGhYFK7rgtCNZr3f7/q+PxoNWq3WeNzf3X24sv58kiRSCny6EGI8HlWrVSgoDaxFfhpLEUl1PAmdJAkoLR00dqsoilQcg9mVx3GMRyBjDLXruhIjmzSNCNlwwJESI/7iKIonoee6k3hyrM48AZryDyQOCCFQmkrP8zzP1xomkziOlAAPtCvAc6QnBKAdGc1NOPV4+ABFGSdJksBUoY6iCASAwGBuDSbnMVGR7zmTcX9psa21vvbcpVrV81x4uBV2e+FcazmOw299+zt//ddff/75G1/58i83mpXJcKBBRdFkZ/vh/v6uEOLalZYfvPT//bNvOI7z2he+4nlet++BrIDTiEFJB3zXcV0XH64SpSKlRCwdKR0QYuppAQDHFfvdXdd1pSMTFakwxndMkkQKlyAPjRKoRmAoOp0nfWB8Z+UjSGV2HCcMQ6z4g0XMURmPogj5By3pvV5vOBx++OGHn//851utFka24UGllA9NkeAYbkyxFk+BdY5A5waOT4uKBCOezQEGR0gjKIqJQcVBpMPR6LCPo1N6a3+A1FbcGwbbJ+Z8RmIpismvVCq57YsnqwfwPTi/eM504JOStdHBi3zXRTfgVhoRSimlEhXHcZEOAWYMyaJVrVaxJoOQ20IIz3Pq9foLz1/f2tp88OD+n/7pny4stuqVoDPfdl05GvbH4/F4PBwMBkFleOHChXa7jez64osvxnG8tbW5tLRkjA8KxQy+gkrrxVyZAKZbEMBhMIMqyHLWxbH81sLBc1cRlPEYlDAMa7XaZDSWUq6vr//kJz95+PDhgwcPut3uP/7H/1gpVa/X2+02Fr2jDmg4CKDiw3iMSX2C9MzC8XGXP0pRkdlrF5EV7U/qQFHoZRGl4xAOeq5Z+B0B8WQycYTQiQKtJQidKOm4tUpVKVUEu/oxo6AQAitsaq01TFOlhVF5aLkCjeoZ3SY+KhEqUfSO9S0YUywY+xhK9Gkwr5769IoUN1Q/ccOBup7v+9VqdWHx6vb29vb2w2o1eOnF569cuey4YjIafvvb346TsLe3++mnd0ejwUJnfnV1XUrZnLvs+z4eK0O1zZaXl8l6JsTB2pFSquTAfcfhGA/4gHSdLMnCRWj/ykNrIJMprgpKu7iu2+v1pJSoJ5HaMej18XGf//znf/zjH3//+9//gz/4g5WVlSiKOp3O8vJypVLBcZ6uTQHUMXp0US38p07PLBwXEVdmgaE2955x7aZI2yXDMf7JE5YeoWMpiyElg6J27HneYDDQWgEox5Gu64/H48lkZAzWRbj7JBQBrbWYvsK0G2SOtyyMzyoJk2GEcGzVbxTpajsUiI1so5TSQtJE57aPRXspqQH379Vq1XFr9Xr14qXVerV24cL64mJHSIjDuX/2z/6ZdAUk8d7ubn/QU0qF41G32w3j6n53bzQaYVn6ra2tMAyXlxdNyMG0ZFV2sjicCVY+n14N34hqvCAWe56Hh5NZ7RwMmvmTXweA0WiEli6tNZaNj+P43r17/W5vMpnMzc1dvXr1c5/73LvvvvvKK684jhMEweLiYrvdlqbOjDCRG1y7otl5xOl+THRGu/XoVL4n4liM/5JVC0mZSkBFM6fT1eUtZjoWsd9O8whQL9bMSq619n1/c3MzDEPc0mJVBOS8p1gtUAgBQhjleArHhMIykxDxrJI0qfYWHJNo16asIACgbZQbKyKl0VhRUjoHtWl8Fp0EGsXjWqVar3c6nU6r3RQSAMD1PI2wLt32/CLWu9jTolJVPlTqjdrW1tbe3h76A11X9nq9ZrOJtjJ83JQhxcFhDtbCkek8jgO9IbHLvNDrWwry9Hq6JgktoiiKKpWKEGI8HuNOotfrVSqVT2/dvnv37sWLF/v9/h/8wR9cv3791q1brVar0+ngOa1UWAOnA9Jcl7t3OTv0CwfHllWL2EtlqgWV2LkADo6kBBbVX7LZPJT0NATzYEsIuFs0HfY8b2dnx3GcWq2GEZf4XCzZU1R57vja8fH6L6ZOpwMFWTAXKIbBGtu3KBnP806kHWtTcZ9/JYyxguoFSpZYjMYKyJSE5kQxcMD4MwxDpZNuNJmEI9eTjWbN8xyhATDAVguttSO0G1RrILUWjcbccKz6/b5SCuV6rVbBSm+kQU6fIqZskIVjQmH6DMzm5piatKQ1J6wAtK0aM7K0bzz5FONJOp3O5ubmw4cPX3311eev3/ja17728OHDxcXFarX6pS996erVqw8ePKjValgqiJ/Ggi2TD4auzGzHZ4h0xl5BmywS5hSoVASvtNmEtGZ93M4I5hUBOIhRJ1NekiSOIxB29/b27t2/c/ny5bqo9vt9DSCk1pAkqsRn9+Q4j7aGYBYw+mFoJeDm9In150kSThxZLR1WKVuw+DACL4qpmBoxRArLsoQaN6nYrM04SZIwHONBy51Wu1KpTOvJCRFHUaS0KyBWkCQ6ipK7d+9tbGxMJpNmsw4Ak8lECNFoNOI4xAdNsVhPe46uY8pe4ZZivAH1ErIApDCdOcZpl2khcu77alPhD1WNXq+HBy8FQRAEwcLCQhiG7XZ7e3sbY5k+/PDDL33pS0II3ECguZmCjsmQSDLyxBvZx02/cHBcpPNSZAX+S0G+Rfsa13UdcygqbUVPMM0UfQwAAAexzCQYaOflOM5oNPr6X/7V7/zO76ytrVX9ANeniQzNb/9xR/QQLkD6fCYLdx7FmHOOiNRGy6krMq4wBA5urEDNrsg4NhwOMVaMNuOI3Volge8ppXrdveGgt1lttNvtdruNOSCTySQMYyHEcDjc3t4eDAaDwSgMx5VK4DgOZhJ7ntfvdzEyR0MC4mCN0AeOv/QikkXNExsnSQzMlMeVm6OMHn1OkqTf76+urgohPvroI4y53t/fj6vRK6+8EgTBcDjsdDrdbve73/3uV77yFcFM2BRgx/Uk4smzzIfnBo75aJIKyTdQHMIAAIMWLeevUgoPPrD2fVktmGupuf1RplgMsALH/H6ueoOp5Uq8S88Nw9Axh2W4rkMdNsV/E/yzVqvdv3+/Wq1+85t/c/Pmz65du3bjxo0rV66srKy0Wq2gVlNRpKeVXxJeEYle1sr+oo4hsQHRmBSASYAoclBPGY/HQoh6vY6f8dB4AQ5aJ3DNKaVAa8/zqtUqHu5bq9UwXElIGUWR5xSyHOo1o9HI9/04jl3fl1LCGQsRnWqm4mBzjQNJn5MkqVarCAQ4m1EUYRwCabWVSiUIAiHEZDLB4r+Kx5axvD5C7Wq1ir/FLTm1JszGDp84Gg9G90f379/HeC9MwZ9MJlhEGxOVms0mQvkU0HUyvVlqKQ7OkcKvqAQHXxEUzUZvhKYPIQQedcpXDb4LKsi0KHBwJpOJdB2D41NWJzZeWlrCgvGvvPLKf/2v//XrX//6jRs3bly7vra2duXKlfX1dSxz/Eu/9EvYSeJqcqhoraMwJNElTFLCzJX32MnSeSnkHtLFATCqPAvHp2Xdz2rf9CBIYzS/DpBa3mDiN4WpXKWUajQan//851955dXd3d27d+++8cMffOvb/8113WazWa/XX3311Wazuby8vLi4WG3OAYCKojAMVaLxmQAqjqf6mpRCa5UkinYAUkrjhZNkaxNmo4fsi5XjcRVpU0O9CCyt9z366OEHhPUj/uosEBdvLHJRFDED19E4PxBn8htK1DoHNIBU6cQ3ABiPx1EUAUjHc6WUXuCLONZK4H4Jn+i6EowZROlYs7Aio73IJDnwtXAdkxKvuU9PKUXH6wlGAEAuhCiKUI9BZT9KUj4P/qZYSHM8Hrdard/+7d/+/Oc/32g09nf3giDAXYLWGptNig/4oA6QDDhBcsATo3MPx9aC5+uZPGxoUiD/SS4cF7V/gpkjrhXMH2ixBRwkm3rU4SQ50I+iKArD0HWnVsg4jlut1quvvuo4Af45HA53d3c3Nzc3Nja63e7XvvY11A4ajcbq6urly5cvXLjQarXWVi+7rguOA0rFUYQOfQ1A4ZnAODVJEsfzKA8KzKEnWByrUqmQlgcmBtbx81mIdgAyXdng0NHDOUqSBM7k2YBFxI2qVqQtkk7bjjkHZpEXuFeXbQr5DaRIABzUlVJT0QtTtUNox5TFQHNcNAnNQlDoV5BSCqmTcOoqlPLgXeI4ljIHvHifOWOnAT0ljbCgD6IwiZ84jnGfQa9MLVcqFeTnOI739/cXFhZardZgMGg153i89qFaFJ8IGv8T+9sfN517OIaCGEkqccIBGsVpLhwfN32jBKYtLLYeRIuK3CPUH6U02RN830c9wnVdyhb1PC+JQiGE58iFTnthceHGtedwn+h53vb29qeffnrnzp2NjY0fvfGDb32zNx6PL1661mq11tbW1tbWlpeXW61WrVZzPC+JImCefSGE4zie58TGE0XH7SRJMplM8CR5MPmmFNERFTA2hdaS/yd3pqzx1MbmSEFd54VIjmpmF+KCnmZcsKBd/lsLbfkNnF3pJ/jBjO10hOUUkSFJEuFI15EAKgzHACCl60w9q1oIoBOmlFI61sYRknCXl+M4wM4I5/oEMSraUshJoJKDm0kkg1l9nB/w0SAPbCOQ1o6TJPE8r16vj8fj7e1t3NrWKlUhxNTgbhTtElAm6CfN7MxGucEzAMeclYFtBh1TeVqkTyuITUUFaoF8Drntn0w7pr7pI6h4TEua6qpoaJYmVZqOYY2iyJUerXl0J4JRf1ZWVtbW1r78+us6Sfr9PhbGvXtvY2tr66233vzbv/2WEKLT6aC5GTNll5aWWq2W9DxQSiultY5GEe7+0OeOYfxa6/F4jDyNwRKk5UHBWXa0jaUPJYPMh4vD8TnSjsEkUpLrVU9PLD74ljYWOLMcYTkDEy5rZk3OVVGFEEJJyMu9dF2XzLJKKS1AOABCASiA9JkskECC8Jo6wRq7w1cHX2gWb4sDBVll30ibmhXIw1isAxX2KImtHQB+Hg6Hvu8j2uI5hFrrZrM5GgwRprUxN9N4lswLta9Ngt/ZpGcBjq0rtB9JzCkAvBggF7/8/uPCcTlMc6XYepbFwcgiRl+Y3o/HmglWGRY3mEophGNlCvSgdNHGiUTO+lqtVq/XAeCzr76WxHEYhsPhcG9v78GDB/fu3Xu4tfnDH73heV6lUsFM/9XV1fX19U6ns7xyyfU8MPUJCZrRTEEhXJiqK4QQbpA7Alw7Tr3yYdKNLIz60bIcnzxpFoUm0r6BAz3UOJQOwDBjYpbs8EbrKyKmhUgAmCaqA1C8cKVSUYCApVx/GoWGWX2pduQ0cyeOYyG0ZEf6knLA4ZU6QzEMNL/Gt5cfUJytdWmeksJK+oxe99FoNBwOsfSPUmo0GmENIIpiotiJkknBF6cxRzQ40ow+cXoW4NhSjcFMpzX3YLx5Os+GcAItuIS4jsNhl/pMmiNtoIRxqYOx2DqOo7VAjZ60UXShgImJpvbxJGxgJ8NPnxVOXNetVPxqvbqwvHjt2lX0kGit9/f3EZ03NzfffPOnP/jB95IkUTpA48aFCxfW1tYWFxfr9Xq1WsU4CgRKMihLKcMCGw+PHknDcaF4w1nAbQ1aBs+sBzyXLDhW07IJOa5awRTeLN7xQcvlzxTWa4kQLABT7KUUSgHEcawFJEmUJIlUjkGlyBHT2hfTnxvmlFJiKUHzFomUqAHkgB1nYCGsF5y+OE2oZtnhCIUOOzNJug7kqUcIu2BK6OFyGA6HQcOnEcCLyCq5QguM2FAsUG8Gx4+dLERGDibbMS0Sbqh9RPwt+jlnC52xIBOq6ulOVlB4A1+ESZKYADiUIgdhvImKcYVrrROVAIAjMY6Hc6TJxzChEYi/5GhCK0cQBKurq1/84he11uPxeG9vr9/v3723s7W1tbGx8eGHH2KMfbPZbLVaq6urFy5cWF9fr9VqtVoN7RVhGEKSv0nkEFy0VEp+eL4Mx2DAiAtIrh1DBo5LmrIEWNHNB09BU4UwZiMttFAA4LrSdYNJFCZJEsdTD54nPeonCKU11ZhPhPBIk02S2CjpB5s8/i5oLuDeYFxfkcnvt0YGPXhorKDMEcdxFByoUPyHQRD0+30AwCOxwzBsNBoYOqnZYdJgtPIiEwRvlhbRsRjySdK5h+MEtHSkEkAhWVNhKEBpM9dSCOkAgAJwzaZ+yg1mx4ROM8ES7YQQqAYWPZownQdskOC12AuJ6z5GjQLPc1AdBIAoiqR0d3b2br75zj/66lfv37nTbrertdpHH354+fLl0WgkHdImtONOD2RTehq0i7YFXDlxkkRR5AsnMTZBiSIqEaD0eDgCo+ZIKR0hF1sLS+3F559/AZRKoggLe29sbNy9e3dnZ+e9t378xt//tyRJms1mrVZrNBpXrly5ceOGX6m1Wq1KvQ5CQJKAsbSAG0i/utcdCicIw9h1/SRJqkFtNBrlDibG3gVBMJlMXNcdjfvjySCoehzFaGssmOvVHmd5zJWmj6coCdAAArTUGscZGU9KKbVWvu9pSKQDQmo/cAESABGGYbVaVSrWWidJFAQNKoemFEwmEyldx/OkcAXoiofx8okAcKQEDVppBwRokEJIIbXWiVKgtQAtpZiIcap7hOMSEgy/BeE4rtZax1prPdZTIxgAmpwFluELgqrWOokBQDgycGQAAKAhjke4UgQ7nJRsGuSOpikgF5+Z1tQmiX5LU+lJR2uttHaE1FqDRkXYTaI48HwAiCahI2Q1qCRRLEEIKXzfVybZD10authDgwZoOssVYzSPNeNPks49HHMiGS6KjQ9cQhKe0haGNNnEHGl+XMN/yWRnFWdg+z7kYDoa58/+7M/+0W/+5urq6nvvvYdx77u7u67rOu6Bkwd5mivX3Fk03ZRFCTeMkKbMjem0vwYAMC67eqNRb7fXr1x5LQzRQNHtdvf29tAAfefOne985zvf+ta3JlEyNzfX6XSwvOHKykq9XhdC3Lp168///M9/9KMfIf5iJ4uwGEwcKxpSUJNCo6HrphA5u62hYTx0/AvouGp4GdyTCYKrxnycaeSFOSfMdV0hHNIxudeOb7F5lAKFbQDLGrVenOLuCQTx/jhSVq9EejvCuZTu4RVa8E+rKMcBImvb5HLAWqdBufN7lPb5ijvF/pwunXs4zkJwrlpKxGfFUm/5TlOxgNxjUdFzRdq3w2+2toHoQPuHf/iHf//v/t3/9X/5X158+eX+/v7e3h5M/WNTvYBu1qYmAH62qsEF3sEOlIsf/kTegSiOybaDMXZ4PQiC5eXlpaUlbCGO48Fg0O/37z3Y7Ha79+/fv3Xr1ttvv621DsNwPB53u90PPvjg4sWLL730ku/7e3t71WrVMSXosoQ5JtOz5ZXSWg+Hw5WVldEwnAZRsdTKErPy09J8SGdX7FACi38oJIZ4TE/ju2UcRfgVZVTSLAObL24ZwIlzWGkezvnW+DDjT6GNFZj84B9IYPNlAuwkPR7Y4B5E+B0Jjnm3ddrOUzTOWXlccr9l9TrLWAzPABwjEfcfUTvGe5CfjA03oTOASeXE8MZj9aTofiu0AwlVpMSc8UGmhmq1+vrrr//xH//x+++//4d/+IetVuu5GzdAqUG/PwnH3J0NRhGm5WSd+4cFD7MoTJ2hpYXXPd9Yexija61HoxH9GQRBpVarVKvNZvPSlecAII7jyWSCKLyxsbG1tQUAGxsbaK/s9/vVarVer/d6vaLFgMFPUko6QWd7e/vKlSuOKwB0oiJCAbStoP+KBvNgVI+92E7Hq4OMRzsPiyG18e+T25bqnAkhKKJGa62ShLMKadM6XS4ZDmymHu8DkWOqEVkwLUSqlgBRyv3L1GfJSqMdMJVSvII7NcjZzML0Q0ePfyiHb8hg9xHh/oideYp07uGYM0FW2Tw65fJuidQ9AWXZeroCTdQasrjnee12+5VXXpmbm9ve3v43/+bfrKysVCqVy5cvv/DCC89du4ymWyzsMJlM0DfCa3Fwu54vHGlC9IHF/3HtietWUWRrZMB2EsIcSIweFa11lByM+dzc3MLCwvXr17XWg8HAcZxer4fhSlJKPIMdw+9yiWro4GqP49itVMKdPVTSKdQUX43X5eAfnpZ2jCNAuwrqFQca7Dya+CchHqso4zjWmg5OTCDNeNl2ss8FZgkh4hE4Im2sKHkF64Mw8ftgLBL6CK7IY5FOU7mqm0vlneEvdaiq/tTpGYFjGuWUGbT4fg4uYAJ9IMMcWDrnVPqp0xYJuo4aMa0cjI1vtVrXrl37P/yLf5GE4WQy2draeuutt374wx+++eabvd5+o9GYn59fWVlZX1+/fPny5cuXF+YXtQlDTpJEAAgAF6ulOI7WOlFKmVDN6a4f+V4IDYD/oV7pBQEBepQoAGO90UJr7Xmu43kaYypAeP5BnWWESAwIEUKEYdjpdIQQWDwB8wZXVlbQXZ4ldLmgqMAMlK2trZ3NzUZ7XpvMQMyOQWjGuGxhDOgHBtljZleeFhGUSFONwRJmwKohk+IJgBVzlHTdKd9myvJy3qArhLAqozewxu0eQkZ15ZTlT81MeYIZ94TZWXJr+LTPTo5mU46wOq0A0YouubmkzxYpOqXJ3H9mo9zgGYBjJD4ZxDG5dwqzE+cgrtlZh9yIdlxBfcRO8r5hH9BYQYWyXNfd2dn5zre/vbCwcPny5cvPPXf5ypXf/R//x8lwGMfhxsbGBx988MEHH7zxxhvf/va3AUBKefny5fn5+StXrjz33HOLi4tkSo5NyST+OEsH4Ve4tw15Fx3ruA1HEUVGHr4+cXdMUdKtVqvX62HcfqfT0Vp3u92HDx9Wq9XcwcEKdkEQ4DmVUsqbN2++/fbbv/27vz83Nzc/P99qtwEAy27gSXHAoApOe7KOS4QgjuNgRLbVK1JgHVMuVWuNqRZgbDWu66pYcV2BGqErXJkQQhhJmvJnQlrtoEcDgIBDSu2AhY+grEQPbSIo6CdcreHvy6lk3CAPi09rNq3+zLTjJ0dcUy6BY2A5RcS1pAiTeU6wg7+O1YdD7+HgqFg6ABg/Ce3E7969e/fuXQDAqtvz8/O1WmV1dfXq1av/5Hd/FwD6+/sbGxs7OztvvfXW/fv3f/aznw0GA9/3FxYWLl26tLKy8srLLwdBUK/XsQ5WFEWj0SgMQzp52uowbrT56qJdsGAxJ9KkMlPMkzSHQeD9+/v7Qoj5+fnRaLS9ve26LpZFp3SV7LhpFvWxvLxcrVbv37////l//7+azebCwsLS0tLKysrq6urq6mqr1VLTQLGp4QIfqrQW7uNNgdUFcEZzOi1YkdbIuJ5IAxXHsVLTLMTEZNZEyUEhVo6/6FqQpv6kNsXghTzQwfkH3GEAU6XxcxLn+HKz6+WgwzCNVdfsaI+sdnzA0pm0l3I4zg5j+QrSGWXrLAjj06JzD8fa1PlHpidIVax0ADGNNMcaIgNxtCXXBC9ZLYtz4fn083uIXyHN9ARYfNlg+wiUgpUkx5PQ8HWUCbHc3t7e2trCA83QplGtVpvN5tLS0pUrV770+usAoOMY4xw++uijTz755NatW3/5ta+5rttoNBYXFy9evHjp0qXV1dVms9lst0FrFceobFJxDxw6AATWBEDgoBGmYLIWjl4URdVqxfIace0JdW3MGcEQURpnupMvMJo+KeU//+f/HAC2d3b39vbu3LnzySef3Lx5M47jRqPRbDbX1tZQLCFwm27DJI5IsJHkQOzLXedKHRTF5iUAMQuRarMxPVOIdNgZf3EpJSYsAKtKkSRJrVbDtG+VjrvAkxEBJFaLdxxnYnjJ4k/iZEgHuun0sFPfHFNimPzDgDq49LD8NC4Q2otk+RxfUGkFbMvosENXcUGBAWutteM4UTgNe+emFYroJ7lOL0KighzpYAqIlxBffTrti7bnV08nV5o6nPV6vaTM/9Ols9inUyFRaoE6C8TxKKtB0BXiePpKKRXH8Xg82Nvr3r+/gaw2NzcXBAFC1XPPXX/ppZfBcQBg2OttbW198sknH3/88c/fff8Hb/wIj5NYXV1dXl6+fv361atXl5aWGkGgp6aGadaTUiqJommyfxoOBO4tHCdwnNFomiaAxIUfR1tuXtTp/Wn5KF29elUp9corr0RRhDVFt7a2er3ezZs3P/roo7/7u79zHGdubg7N6IuLi4sry9KkwlPBmiiKMJAOjNEADHKRJRFRhoadPIcEE4gvcZTqHmIy37ZrE3xGcG2pco6pK8/k0HRnoIqTfVFe6ux23smxEVtygvdBp7ePqjigM/tD/ifJFc2qEqIcpbfQjOj1szOexdZTJwsKziwsnHs4zl3b2Q/EH09rHvgagLw1YzElbjaLSEpJlUJRMez3+/1+f3t7G5VrPHTHdd3V1dWlpaXLV678+m/+JgD09va2tra63e577723s7PzN3/zN9vb2wCwuLh4+fLlpaWl5567Uq/XFxYW6vU6GjcQkqiMhkwHnzYaDc1MxlkRaCkvXJWGzARlaTKZoJLred7S8vLSysq1MAzD8H/ze7836HYfPHhw//79zc3Nzc3N999/fzQaVavVubm51dXVixcvYu26RrUmhMC0CK21UhoSpbWWjiMF4HkrWmtlOi9NUQUAAY6x1WI1QK29oDndpEPKektzqs0OgGabLD84CDz5E/U2AKnMlq5oHGhsbWTJ3AYZVZEgWEqpVcoGfcS6slldAdJoa2kPnLeVOTeHYzEJPy4Csw86tCdHp+wqO4N07uEYCvx4RcD3FOHYusL52LIgAwsRoYUEZgOrzWaQyrxR6gcSRjLgDz/99FN0kTUajXa73Wq1Ll686Pn+q7/0S/igYa939+7djz766NatWzdv3vza1/7Cdd1Wq7WwsLCysnLlypUrV660222KfEjMCfNoMBmPx/wdaRc/Ho/zN/sZLC5fIXEUImqEk/Fw0Mewikrgh8NB4HvXnrt6/cZ10Ho0HO7v7/f7/d39va2trTt37rx582dxHNdqtXa7Xa/XX3jhhbm5ucXFxbm5ObRdhGEYxaEUDlflCI5RVyWIJPVfHQArpbcdhLoLIZRSzM14gFA0p5hzaCY0J6mhnH+It6nbuXyV/Yw/VCaIiBmmUu1bk4L1UjSzL/OB4lw69ceIVG1rVSu41gAAYLNJREFUzepaQDpisqjDJcxwArIeBGlwOGt07uHYWuSWBsF58ekKRktCAGNNlY7Ax+u4pPnP8UO1WkVMxHYoDtdywZGDa25uDu/f29vb39/nznqEp/n5+edfeun5l14CAFBqMh5ubm7evXv39u3bd+7cefvtt4fD4Wg0unTpEkVuLC0t1et1z/Ncz1NxBZ9FYWr4dB75wF//uLOArxYEgVIKxQxGW5MxhGI/5ufn5+fnL8srU/gOQ4zl2NjY2N/f/8Y3voFcUalUOp3O+vr6xYsXFxYWWnNtx3Gk44Cxhiemvv7UKWf4anrdTWn3hMtSTDOJkyTxfR+mHHjAiiQ1KbpcTYMPtdYayLFcUJKJG1X4vwloi51oWLgIJKGeQMIjvVCs8vVCK8U0m4qfEQcuhAM7D/8gwIY/GkDeJa4sG/GW+kkRHZd/LCnCx/AM0rMAx5CeJGI+zprs5qc2E5ybIcOsRSzC1xgAkNJGGgo2O5lEhCDm/57WOkoSkK4jXQ8d8WZY+v3+1s7e9u6+lB8j3mGQ2XPXriyvrV+4fOX1X/01AIjCsNfr7e/v/+hHP9rc3Pzb//73//Uv/n8I5WgNeOnGtVqt1mq15ubmarUa4iDmOtM7so5J0u513tY7S1T3Gck18bnkc8MnHiCOOw0jq1arjUZjfX2dbsOMwfv3729sbLz55pvf+973oihqzbWxoOj6+vrS0lKz2axUKjyQgPxgOHoxJXpoSonGgLDpFidJEg/L+eupm02YOAR8dypno7VGuzOAIPYthxtLNc7eT+KQ8zwHU3Jqcf2ADMGW1pLtDCnCkhVCkaxCoU4fR0C9tSaay2bO/9TVohEoGp+in1hKwHHR/AnTuYdjyMM1zWxSnCGeYie5nkhcaPWKr5DcimVaa8dJaW0ckfnqojYdU1tWmTJaeH+tVhMs9XYwGAwGAyHE5sMHSinP8xqNRqPRaLVa9Xq90Wg8d+0aaA1CxFH04MGDd95557333nv77be/+dd/KaUMgqDT6Vy4cOHq1asYAT0/P0/QgKZndCGSS+2IRAF5pNdjHEitViOgIXe/EALk1C1GBezJx4gWmFdffRUA8BDi4XD43rvv7+7ufvjhh2+88QY2i8aNl156qV6vdzqddrtdqVYBII6iOI5VrHzfd11X6INDxIUQaF0GU+kcpn3TXAGkADVtoibMrAmg6MYCLuVMnotudIMygYmQQR8aJWXijvAiZYRyfOTImIuSOkPA0B8YFvMfWi3DMeH4xGSJpVNv/1ToWYBji4oULjEtvPk0tWP6rDO7QkgzKIdv/sPxeHpQObEvLi3uk+HNOqaWIxjjBq/oRkR/SmeKd3t7e+gYxJC7TqcDANVqdWFhYXV1dW1t7atf/arjuhBNdnZ2bt++/dFHH3366aff/OY3+/1+FEVLS0vtdnt9fR3PYF9YWOjMz4Pj9Pf2rNcsF5Pk+iczJZqPwUSYaFb9VikVRwmJK9pGYCMoD3Ar4DhOs9msVqtf/eolHBYsKIq+wb29vW9961toGHFdt91uX7hw4eLFi51Op7Owhs5Socn2qqSUSRzTs3hUCYdj2tMgHAshxNRyLbQRmZ6Tz5+WAnvAPCLHPMrlvcV+5IklwQwZ1M5tjTeiWcQeDa+eukBS3bMUhaPg4OmqTU9dDzsWnRs4tjBLmlhj35FxHCeJCoLAcV3yvSRhBFPDhFbGHieFKLJVkO+LwxOUxDMWXyelADKhx3QbsXWJVpt9dwDwPNSLE7qmNShVGKcZTWLXdcU0xEoqpRKVOFJKCUol6Gd3pTAmQuVJTzoAZjedJAlokNIZ9QdxHO9ubd+5dRsPf6rVakEQtFqtdmdxfnnttV/+CuJ9r9fb29t799137969+/N3P/jvf//9yWQSBMHS0tLS0tKLL764srKysrIyv7QEAINuV7AjAtKFdbTW2vMPCmxq0KC1ACFAxHGilAIpcLeAeK2VdqUjhRRYVd3sCaYhYUI6jgQN0SSkcLXJeIBTUAnc2urS+tqy+KVXhRD9fn8wGGxvbz948GBjY+OTjz/42U9/PJlMhJqe1X3lypVLly51Oh3f91USNZrNYa//7W/+t5/8w8867YUoTEBLAdLzAeMKtNaYecgTcJBdpZza/V3XVXEILAiP7vF9nxhSmaBdFMNaa620ABBCuu6BfRZACD0tsRbH8SSKpZRhEgdB4Aeu1tpxhQYRx2FQ8ZA34jjRWoHQUgoKMJcsMxtMrJ40gdXcaqGUEkJSZXohxHQnwarIktGZzNaSpXdy+zJnfvrXEfnYSi5Ei+JIua6bRLHv+0kU+64HSqs4kUWFc5+q3nxu4Fhk9vhT4HPyDx61FMwTP/e0RGtRO09g32Rxdkl/dDr6lQsn3FyjpqyUQoMyLhsMsahUKs1ms9PpNJvNy1euXLx4UUoJjgNabz98+P7773/00Uc7Ozt//ud/LoQIwxDD6S5evPjKK6+srq7Oz89TB8j0jKko+BQSkOTJpD7zra7nelnYogiQrKaGdTCsLQLq4IuLi6vr66+8+iokyXg06vf7k8nk/p3729vb9+7d+973vvfNb36T48vW1tatW7fW19crlQraZwHAcaXVT9InuAeMwMgS8jRNmABJ2EcmIErB4C+l01YsODBVH2RF6XQ9JtLfKXeGNF/BashRh/lgalbWlYJJuGWM7xJ02i5R/uG0iPhZpvN3zhqdGzi2iBabdqd2Q5pya7K5rlpC1g3U1HFhtKidw18pTafOkZbSwa9nFwzvA19CYJQjvpMYj8fj8Xg4HG5vb3/yySe+7weG6vX63Nxcs9n8yle+8vr/8D+AECDEeDC4ffv2rVu3bt269c477/z93/99t9vFuLrr169fu3ZtdXUVs1ocx5mEQ3KpKeNG830fNU1a7fgtAPT7fcRfDO/DzmPQce7AGqOBEOysQsJ9MI5Ex3GwItKF9UsAAEpFUYSB3hsbG1iSf2FhAY9/bbfbyJ+e52FVCm0ym4lXjzg7/CtyANB1bjHIXQUkwyQLSuPzSz9ULBUeMlZmUn61yW7lrZERBgBn+OAkPcujaL1jtiflPHwCEmmTIA3I2aTzBMeEsyS3IV3CzeLm7AQfCq/8/pMBYlE7pwWvxxUPVh+sBaDTfiGdKWdOK9AEZh0czo2Jar1eT2sdBAGdVYy/wjOAHz58iKhUq9WwkDEecf3cc8+98OKLICVoPRoMer3em2++ef/+/Zs3b37961+P47jdbi8vL7darS/80ufxnJFWuw1SYok7tG5rs8/lMNFut9GPh3gNxsTM6/Na67NoPKllYOpe4BqjvON05uc7Cws3nn8eFct+v9/tdnGs0KHn+34Uj7jAwEYInZVCz5tQrFCJNVnYDWyTqmNzEwG/05I6/CuaXw6R9F6UJa+NZo1ER5VnG+e2FMWKwFCf6RHZnlA7lnjg10v4uYjDc2eSjypF5hyr8SdG5wmOc4lYCpgpNjudOq0DZolrf5BhlJL7y284tJ0iOi34ptZ02kgHaXSG9AK2ZJ7WmgecAQAmg2itqZikMilniNrI91hYjqAKY3IRyqWUtVqt0+lgfspv/ZN/Qv3Y2dr6+OOP33vvvfv37/+n//SfEEQajcba2tr169evX7++vLyMBgF8LvrcsP+j0UizrToAYDfoTUnvw89JYudB4PuiL0Gycij4XorVg+bQ5rpus9lsNBqO42ClDillGIYgDkovcY2SoWhq8DlX4UUSNqQa0ytwgUEkjK2GPvBplY6k1cFVGXqcdcUxVQbxOskSmYmDphvAnDFGdS3IGpPLmZCG+9PlfDAQQUL0zGIxnC84JrTlE088ZH2WmbhL/m0RHV07Lmon+5STsVc5+x69P5DZT1jtcEGlje2Y2qTBpELGtLqEiRCgFQhG+wCjJmuTQIgo7DjOYDBITFHjvb29hw8fojqJtYHm5+dbrVar1fril770xS9/GQAA1M7Dh1gOaWNj43vf/f5ffu2vhsPh3Nzc2traiy+++MILL6ysrDQbc4jLoQ4d05/JeKK1xkgMFliWOpE+Skyk2nQ4puMTVGuIO7FS07pBQkjHCcOIjKIJTA/z1lr3R0N8UJjE4yh0HEdIEUWR50x5j2vHwGzHpFdOUcPUprCmj4aX7DMWEHO4FEKgWRnSVSmUUkIKgnXUiF3jAOdPFJRlx1YZNwRzfqPEd2IYzU6bBAbZkBGHKi8sT5+edizkQbYkEH+eVUQ+T3DMiXkJUhoHzSJnI66KFG1Oc6+fgC1y7z9BO6dFtDyyq4iu6PQ+lK8NuqFSqeDKoQqZOP50KDK6/pXJA6ZEaqytMZlMhsMhmNI8CM34rMlkgg/Cshtg9NlKpeL7/vx8OwiC11577Ze+9CUAiCeTra2t/f39Dz/8cHNz87vf/e6f/dmfCSGWl5cvXbq0tLT0/PPPo7rd7nRAyngyGQwGo9GoVqvxYeEihw8IfcbX5HooyR6OSjRcdPQ41lTCisye5+lk+nZcx8wFIMjDFP4UAkdu5+Vzx3srWDADGNcc2ZppEOhdsowBGd8dX0S8QZkJmJMmP57vUSzKPtRq5BSJ3rpo+Z8ROjdwTJNEszvlpDhlcaPPksV+At+sHW0+rMc9erdPi07cIIdXWqXZe6wAO9opY0l1xYJqkYQQmBSHWdEYAFCtVoMgwHNAlDkhG2O8yABK4OI4DmKZZiqYUqrb7cZxvLOz5Xke2qY9z6vX661Wa3Vt7cWXXgIArRRaqG/duvXRRx+99dZb3/rWtzBYeHl5+fnnn3/hhRcuXLgwPz8vjMWACF/N811rrglelYkFBoYv0eTghGY+dORmxIjmwWBA5gK6gT/FeD5s2zHnTguLBTPU0rjxynC8V1xsgIFjKaV0HNKO8R6Hleu0ngssQlmn6ywDE+ccnYGtTW7qkelgDP5Ezk6PyN5ZEum1LKU8s6oxnCM4tmYRmVJKGfg+ACTswEdKjgImz7PrxyJu58IrfJ9VQhboowLFf0W4U/RcYDLG+jdLxxXvqI0KsyukISJ0INsi5C0Mehxpi9a/VHkOBxAfNx6PlSncrFgcFZY8FkJgeBkRWkIIiPEerEuXJBFGdOGdeCgqvlelUsGMwaWlpYsXL/6j3/xNECIaj3d2dtD0/POf//yNN97AsLmXX34ZS/Jj8Q08aRAtMzyFT0oJILTWSayklFJAgjjrTCtQc/0U70cjSbVaDcMQz6AaDof4OmEYVgO/3+9jASYM73Ndl8ZHTIsTuWDe2pEOnUEFRrxRiAiXWMKEPXDoJMVZSkkxi1JKimlzmMmbsmmUOaYEhwKfiPNCLlBhihFzvrWGwnGcKDyog0y3EWPwZyHPkAgkZNfFBnEhhErSFU4NFa0LfCL6dcMwxC1aHEUuc+2eHTo3cFxEJIdpBwRpJxVti8oFL7FFrtpYRNnHUX/oW96xp0uPqOkDs11YTWWvUIQymIQFPkq5LdO3fB8jZT6LhmEchv1utw+wgSYRNG7U55qNVvv1X/01rLkxHo12d3e73e7Nmzc3Hz784T/8ZHt72/d9PAf2woULq6vL1Wq10ZpDyTGZTCaTSZwkniOFFNJxfdfRWiuAcRQqpSRo1wR+AYAGSLSKVZJo5XjTQ7gTrZTSvuvUGvVxf79Wq2HjURSFcTwej+fn5/e7Pf6ypP/CMQ9fTdgpIRwZuV5PKrDCYvsG+hNjp+ZREFynhmKtk+YXO0+OStTv2esAHObY4Pec+jKhNcif8ugL4THRuYfjcuJYnIsjRFz7EIzK+SMLx5A3908RjrMgeLKe0MjoAhuf9SBpDqEQzOJp7d85Zfcilt/fIrqulMKiRcPhUAgBWw9pd1yr1dA3uLi4+OKLL2qtoyja39/HgqIffvjh97///b29HTxRZX19fX19/cKFC6urq/MLC0kco12F7AOosEeTMWmaAIBB0KgRV6tVMphGUTQejzHsJAzDyWSCxwZW6/UgCLA8CBjVUgjFhjentplOkzVupKTTtzytkdjP/Fhz7YQ+UzuW4CziFn7IlmQUR6lSsRYaFk2lJQBOcb1opnSDUaKPu8V8YnTu4Zi8EzTEfNyPiMVggIY4WGa8E7lEPM3vzALWCXjruD8puj+7hukdT/BcvlT4NpM3iH/SflOzCPHy9i2w5qso92a0JpNZE9F/NB6TfocHiNy/fx/ZA/MG5+bmXnnllde+8AVsZ9Dv3r9///333//444/ff/99OqD6s5/97Nzc3KVLly5fvtxqtZIkwXhnyvqlc7vxudVqFS25GOSH2qiUsl5tjMdjPAY7iqJ2EARBsL+/7wcVGhwhUmXQrEEomQ4a8GwAA6QlFo08FgiV6XgJzUoaSXasVMnTUTvWzP4wVcaFbYsvkanA8DoraU6RaCuQNZSfKTr3cFxERXrxyWDo6KTM+TSPo/GT9Qc/cHkDx+dIvmwsXYZgl19EEySYmCcuI4v6yQUqKY8l2jHqxfTn1HHkTutXWCXxkiTZ39/f2trCNIpKpVKtVn3f73RaS0tLV69eRWPi7s7OgwcP8CjYW7duffvb306SpNFoXLx48caNGxcuXOi05vCH6F1EOEYUxsdRASOEodu3b6+srADAeDzGPMMkSebm5kbjCR9bzT5bH4DJvCxmEXpyHQKtLoIZ8QSdxqvtwDKSplysHsqxdBtHUvpMz6VOHtqapc4X2QxPgKOcl6am8BkcPyYqsh1D2nVr6XRZIv2CCK8fEbY43NPPTwZ81GDu9eOKk9w+lCy2knaKJBywhU33c1MD+V3LF3m2QSFEke2YsN5YEvAnut8fUg634zhSuK7jAsBwOBRCVoJpWdEkTvZ2u0qp+/fvY4gCZgy2Wq3Ll69+5uXP/sqv/joADAcDPGnw3Xff/eu//pudnZ1KxWu323im9cWLFzErut5sor6MauZoPEYnGyTJ8vJyEAQ3b9788MMP6/V6u92eTCaNRoOLH4G9R4tBgXZsjbAlFFWmQhvnQ76DTFSStR1Pi4IeGE8OWSzAXOVmmox3Mcmxjx2qHVuTforaK40ApiBN4Vjrs4nI5x6Oi4gzqzjMUkG3WaxwlF9Z5LDDcTmzJkc7lOzUKfsKJ94V5iJybms6nWxthZockcrVNF5SRxh1OIoivxIcFHjTOoyjcTjBkvAaIErigz4LEI4MTLj0aDzu9fv3HzwgRZKOs/rK66//+j/6R/jc+/c+3djYeP/99999993vfe97Usp6vR4Ewauvvur7/qVLl9bW1hqNRhAEIES/16v6zs2bN7/73e9+8sknS0srC0tLQggWBnfAc0au5GvH1r80BeTK45qpNHX6pTmBie8zOOASe1D2HRSEo1mE4eS8CBFed6RnKcsljUBGclivnMMVRQ0VEO0eUORQWedTRPxTpHMPxyW2Y5HZWZcQsYVMB7qVU1ZJtJQU6slx4fi0tOPcH55AO6ZvD+VjbbxJtLx59eEj9pCo6CcqHZ9HClqopoGDVmWyyWSCZgSM1iIoQXMH4hdGd1GZ48lkcvfu3Vu3bmGKNp4S0mxUr1279tprr6F6tb219emnn25sbLz77rv37t1D3FdKLS0tLSwsjEajux9/cPPmTYzGw6KdjUaj2+0azDJIxIVEZmtVMimoF9PxJWBCpPH8KmJpacrwj6NQ59mOrWTo8kmBdJF7wX0tKmdDdsQ1aL31qRBpx4qVptP6qZ2ZWU5nIgDrBLS7u/ud73xHCBE4U1YTaeOXSBsu6DWpuAHxEP6K0h/o52AMlLkdKBo3na6GbgE0pI2tJS/IT27mkqYI1kvMcxTYwD/z8eFdKloJ1rDASdeMYnWIZLqeb1HnRdqzSkBAWjAl+2I2YG47FA9O70tKmTUyUDqYCOtoO67VanhmShAElXodkmQwGGxtbd2+ffvDDz/85JNPtre3e70eRn2MRqOlpaVf+7Vfe+WVV0ps4lmAQ/O0k67PSz+P9YFvgM9LEZ8QPPEH8Z/TBE3bn4StVkuYGPMwDDFkm9vH+XPH4TRCHPMqUf7hAY+u6w6HwzAM8eTD6Yc41GlXpDKBybnr97hc57o+jnYYhpVK5XOf+5zr+1NLRaYlfSLb9CnSOdaOT7bjIAtmFo75xHMP2LHap2wu3r51+M0R29Ess5YCiU7R6GHtJQmhTqv9kudC2kxf8lDJ0r34RV5tElvQzGeYS1z8QF4ADBxhguhQqMFg0Ov1Njc3URtA5bperzebzV/+5V/+1V/9VeG6ALC3vT0ajXZ3d2/fvr2xsdFoNHjiT5aQf7TJP+ZclH2XknEraV8b3yaXTJQuJFk5CyklJAd5JdKkJmIjuY9AIwbJe3zZMAx5FafyHv4i03mFY67TnezncBxwPBXKSvtDtVG67dAosfKHQgZ0shpu+fIo2Q3kXi95Ly4RgSloJUSGDtorUASLYJZ6St4r6SexjYVxueOTJSydzPc62DFM98KKSJId1bq8vOw4zsLCQrVabTabGByNIRm57aPWSXKFEJm7znL7bFHJ+GtGXBzqtJUPsBCHSlUZxZdyHIfK9tNwTfUYfcCo1DifX8F2sU9gAXK1+uwLgHMDx1yXEcVOpEOJMzqkmQNvwA+kdh1XG8V9HM9ZIm0C8hC5pJ8iHbqEPXeKDpUppnLt74g8elorh4sBa8xzidsowcAfjQkPqKL4jUP7zz8XgXLR+6LrjIxIxEtYoogjHQLWe++9h6YY13XDMJRSoqEDiyUV9dPaOnBMsXt4TIRRphAHPYibnrlsw0djZB538WljJrK6bYl2LmvJKsVf4YmBo2YVM57ME09M5waOT4us/emhM3Rc7e/QdgiLnwxT0oOsf60ROHE3TjA+1lflj6bTOnhkW4n2elyxUbR7KLqf6jsDAKXtaa09z+OaJnV1fn4+DEP0WPi+j8A3GAyK2idTBn8KiRmui0z7eSI+5HAJxieh00YM/OwIgVd4xU5dbOfx/QBbIxmJdTDQiAFpOH7y2rE4WpDV06JzA8eWJmWx/tGJ12Y9iu34uFRkOyaXLhwt1Iz7yrG35Lk6WccssjrwJFUVrs8C07nK7+cwp/MyJ3GIisYnuxqpzSMCMRJOJTmyyIrKD3nCO7F7w+GQB+QJITClmw4wzb4vpMON6X2tHhouOt7ECWY6wBbQVUjXFSvfkySJlNP0Qm1ixXhYm8ho62RUofflFfv4FDwZlju64nUW6NzAsUUnhmOKfBKMShbhY9KOS1qmbzknkSvvuHBcrh3zt8hi0xHbP+799KCjLBUyzvCbKdFDpqvukk0zl7IMw8XbEUGZqtNZXaI+WLiJLjKqKYpUq9VKxAbfAXDJYfXwUP7JvU4iEIGVnoVHtAhTLvUgOlM6mPNNRe+4USirrdNhTmRDx4gX3mF6NaXUE8BIYvWD2T/+ruLJ0HmFY3iEQKtcZLRCKcjYdFz4K7IdW3GyR2lHZALdTks1hrT7hfDxCWgQRbpeEdEugbpKhki+uQEDMUeE4xO/LEkCCxDpKFWyKZPJFZEaCwxhtc8SsSpZMTZ62RN0tWhgeQV9sn3jYGJENj4LpSBiK2ZPkM2kqDPY1EFgr5ErqH1LnoL4pJgNGBY/gWc9Op0bOKb5o3Bu3/d7vV612YBMZQb+QxLjFL9JLE7Q4JiSiRaV9AcFPmTUTwsgeJ/pRXg7CDe8ADFt8Xi3sx0gvQnS4QpWf6hGLfdxZeGAflLEuFkXmTbe9tz7i+AGJRMJG2ECh4vaoSBlylPQxp+JyEvojCbaov6jSwoMnlIldciMDH+73HZIswM26dVqlTrM9/I0DlQwWrIaPZAx4/KIBT6M2VC/6XUxzbjhh2PheHI2oMchsFoOScTZ4XBIOj6OJN4spURhg7ozavrValUYRzcqH3gnD+Mj+EYZQOhMIyyEiKODYBg+Jll+tj4fkYQAFNJKqU6n4xSEt58ROjdwzIl2ptIUcoW0UxsMzOmMw7eoTStGiiNp7v3Z6yT2ix6R+5WllXMoL2qnvHH64dPVCI7y9Ox6O1b7lkAqf2h2SwSlk3VoO9SN8ucW0XF9CSVkpa7gn0n6WAMuZsp7a33LDcokgIWJr9DsiBBzQ86A5A7+oS9VpO4csYXzSOcPjmkzhT5u6yud3gjzbB840VwWMZDVMhwGx1kN2oJ7q9snCGgj4sNS3p8sHXeIyreuWbKW2YmRqAiOi8SnTueSnNjso6zSP6Ynp2hHOkGXaDuVNToDGysokEbWnpJ2eAivfJpoT6PT7jjabgpD2X5aO7Dyec817h0qS847nVc4plq3pAUgWTZfUpAPDTwkbdqiovsp78jivyOCCy0AK77Y4r+jE60WawU+7tJFx10eWdv9ieE42wFxBK9s9t9jUXb7UoQ+5cQNGpyFTjBfpLQCs+1QaQ5gkg8yfGI1AgzB8SeJsbPxFUSaBOoN5BV0HCfRKX8mwXR22I8+aI+iTp0vOn9wjGTFrtN14rMsnh6qveZqbSUcoNOZRUeR+UXatPWrJ8B2T4uz+ahaQ/dkHp39/AtFXKu1tGOuW9BXZGXmlViUUnRYOAZTo0HZ8zw8YxDSPttceXCUlZJljKe4C3kCdG7gWKQNEcg0WRcZKcsWexGLFLVfZDsu+Yn16KOscOsV6KJKV9I6ATzx1kTG719y/yNS0VNK2rfG9mTapaVi0+eipkrEcFEni65bHT4Zsp+i7ZjWArBu0+keVp/5bXyNcJizGFuYOEu6ga+sIuJWOL5V5S2UvFR2pfD+P6t0buCYExfdVioBfbA46SgIlUsly5L3JDd8NfdXOmPag/TCOBk8ccpdWrn9z9Jxh+hkcIyeHxoNeVhl+tznZve/5XDMvzrxCFthOeUPfTJEZgpgfrYS8WPJEouTiTOVOQyQcJDPF1ZoE6aGPfucAm4L2YFBfPmMH2V7+uzR+YNjvvGhkCkOu8gN/OyvQ1VjKLYdewUHgBNr8v4ci4jvqQWROdbkWK1ZqpY2ESYlT8/So6hpR6FTkTdwfDi2VC36XJKOUXSdT5CFYkenU7QdU2oGmBJLGNDJ+cF6TQuyrc7zP/kxAvQVRsJhrNtoNAKASqUihJhMJtL1s0NhYTEcQYbxAhenwjDngs4THNPEoGSm2Bqsz6JNiC6WEaAqJzyiHiMQaWq5d8IK+6WHWgFDlj5LPEqdKVreVGCTBAmxJkGwZKm0PDyzfEwgEzbAqURTptfRJs+CL2nN0lgw8pTe1EIQnZemUdRtCpyCdJAv1SOm31JILD1FsHBs3/cRg3hRzXK5SAIbDBBrrekITmAh6kg8dIx4D8U/T6PgWWq5ZMld690hLVGwFiuCKRpnMauIGxn4v0kSkyDnT6FZsOaCotP4+2LFfb5q8E4p5Wg48jwP3ebKEP4WD2TB4sXj8VhK6ft+GCssp0mBT0IIzAinfD8wERpaa6WTrFjNJVo4RZOb/zPXCZNEaRXFkVsJNJRp3E8d8s8THHNGIZay8BGY4YJ+SJZZC0+tD7lU5KEumv6SZUm9PVQ1gLS2TlGfulhbL6LcZ4mMB5K0HovphfHeQIGRF3tF10uAuJw4FitW8Jd3kp4I6QLqwCaoaGAtYLImVDPLCV1X7Hw/kpfZlo+rux06RBRsgweLJEmCpeA4vJ54nIvel97CiuSj+Dnef2DDzsccxZs0x0GA2b8qpSgdKTsLj5VIigshsMzT437io9C5hGPBzoCw1jD+SSlYFlnTfxQba3kLJZ3MvZ5FLuqzJSco2bcECI5CtGnlnc+Go/Cf8F5BgaeRPqt0IBRh5XE7bA0C7wz/ioso3nm6XmScsSZOZ8j6ysJiYOeS0D14P9f3S55LTZXIDGBZi6gUo06KKrmFobmvdih/Fr2vlUVCV6x0f9oFSlYtBNgpAcpMGQ0XVVwiojVb3tVTIRp23/fpxJnyKXiKdG7gGEdQsdolkFFMOB/whcSJr17+oQg+LA91bptWP0u+tZZTlilpyXHjBl9Cx4U5yZK/eYMiHYttDSMfXgvmcrGAh5qqdIXiIxK/H/ssTRUbaQgMWFjCGDLG0PL2IY1K/F/+Jw0UMKFIv6I9O35V7vHnTy/nH9d14zhGjRKMgYX8acB4fvpvXrNHYdGsnOAGN2qB6h1bi4sPOPcfKn1w7hcNmmSn2FicVtLP0yLsWDZr7AzSuYFji4QQuVl5RDIdiw4nhVGq3Gr9e0QtrOi61Te6h1gZtVricg4fxyJrl0CLuRw+RB5BARafCnF0sKCw/H7++QyqPNkhIoWg6H5CRqraw082svhQ6RyjzcnGgTMGtSaEXTOovHFclQTQyH5WzhSf68c9YcJUuEXVGKX7GeQTpHMGxzSOUsp6vc4LU0hTDkawvXP2h5ytdXpLnvvEbHwowWXu/UXtWJZQrnbl/pzioK3XOa7nHV2Xgm0e8U+6Ts/lI2lpc9wDZrWfs8BKx7NoJXC3HlfH0H4KbAroq6xuW7LMLLlifaUzJhHrZlIb6XH8NUvEktWxQ4EA/WNoaaVintyYm52I48Jx7vvq9JEZwhgAXffARMNHhuu/fE1lIyIsjsquxMdK9CKSFQJ8As89GZ0bOOZciGPaaDQkK47FmQnyNqe8HSQ+MUXHrHHbyKEqGxSvBOu34jDbGS9BwMMeyCVyRLJ0AeoGJ0shFWkLqSi1XFvePxIexw3X5/3kkMfxgm6mOy3r/wngmM8IBzuZiS9W5nQ+/ixdHNNC7yLSgp/jZm4/Ic2u1hXeIBQAffk45L4vLSLeMR5Hwd8IjQ8cZ5UpBRfGqWPCyXbMC58KJkoft5pKYmNmO35chFuPSqUipdTxQXwFl+Gcq/hnruhZDJ1LfGFYiJ9LRchlxSpAWsBkFyfWY+TdOBkDIQsSplNYGB0mRO1rYzSkdFjqp7VKs8iYjds7gXZMAGH50HIpK0IIPUuea72yhc4WtFmhY2DgyfoVVy2Lunr0uaMCmMgwVAMTj0zlWDx9U+fYLtPc97XEDF6hGD4y3HPtmG7m13lkBXIU2cGp/SejFyNpE1lB4bAwg+PTJdQcW62W7/v94QDD0QEAuRZj4LMBavhbYQJO+QLGQByqGKuNzYtCPmn7TG4lXB4Wm0qTbEZsCgVaG3UpK0LoOtdfCHqKjBVFcMDvF8zkTS4aHsmELiO+4RDG9CaYyYIPBe8w3U8vTj/RLFyPHPS8nclkIlgujyW0+DzSdZIcwNC2aHzIfMlH28IFrudqtsOl6iiKVdrj4geYemi9r6VX0r8YO0+lirXWqEUS4/EHUZ1uPm7YsVE40abWpTZFh0l7BbPto+OROPEAUHpfPOUaa1AQ7vO9o1IKLSp8Tg9GTxwcKkb2Q9d18TCRJEnwt57nTWM2MsUJ+J98xMqJb874ipOOE0XRaDSir0qKYj91OpdwjCSlrNVqe1sPx+MxV+v4koDiubRWOE0n51FIa2H8T87HHDuOwjq53aCfi7T6Zt15XCr6FZ2QxpGXQzN/QQu8eA/pCh+ucnZXpgY5gTvCTe6YY5l/C52hePkVEUkO6+2K7uciAY6gT/HO05VD+0PASk8pfwvrBgJifDoZtbLhQIe2z3dRuASmrvK8Xwhj07e2UyV2G5GO3OBnXD1WwidWKhU6H4DUiDNI5waOs1zrOE6r1XroeaRVSVPmDdJaoQV51nWVSSjQ5ujy3JgtC47p+lFAoYg4f5BSY8EBnBSRc4nGUzN1XjPVFfLeiKu9pF4BAztrZKz+C1PugOuSytRm5POr05l+fPYJxI/1vpJl0AHT5YvaKbKJl7RPPYe0MMu9X3D7aWa4cskaTEizdJaHsVlucjnUkaXTqreUUkUHyG69r7WLmk5raZ1u6h6dD/K4cRHfqNFoNBrTY4POLBbDOYJjTrQ25ufnsbgfpszjhkib8u0cwrKQyj9rrXHzCGbCuAFUMF+WLI6fOxkW01LkHwAAN7BHVG1KqOhXPDmYa6MibTXm17OWU2usOApAGiaAoSo/eJTu5LdBRhJYiAxpHwD/t8hYwV/qKONJoE+3HWoTpxGgK4fez8czK/hz38L6zOt9Azs7yorMOfSVufENaWqsSPeZf0uDU376H78f2MBO+/z4zchKKa4dn2U6N3Ccuzjn5uYqlcpoNCKbl06f8pv7b3blQzGn0qLKhYlHJ1IwuT4l0jYQKBAqj0KaacccfGlDIIzdhrQqqw9HgTNIa23UZjbXCwPvJHOWkj00t9njEqWNcDX2jOtKFmUHnI+tZjEwp8gnRBb/cxcfajMoaIsgGXNbpDlTjeZdxY/9eAQhBBZl1ukd4WN97sno3MAxJzJCVavV+fn5u3fvEnOg541rB1kstpQ4mQ4b4J6ZLHZDGo8snegEc8ydzvyDtaJ4t4/VftHKpJI92lh+IE9iQUYScJiGPMUW0iPDrwhziiWYNYz38K0Ml5FW/7lcPK7tWGfsS+XaqDXaJXou72f5JGbvlyzN79CncL4lMUlbOs7bmGOd7VjJKHHbMV6ZGivUgauND6BgGXrUMSllGBdWyKPb0IF56ppNEVUqlVqtBsxSMYPjUyPiSDQvLC8v37lzp9/vg9GALP+49S+3SIr0gbXcyUPKVNEWjAMNrajyFZtLkkVW8AaP284JyNIUhAmSpzHhmmPutprjAocJnad688Vs+dMohoGjmIVT9OFka4ljBzVVohpnzeJQanm0PMCQAXSLtMkCBxapcizQn7bPvuLjLzNRLuXviyTMQddQYKzgcAx5voSiljG4Akx4D3bycZ8cBgBa60aj0Ww20WlBg/O4n3syOmdwzJkVDcSrq6torzjYLuV5qznYUQucvbTZO/NSahR4xBeVpQcVNX5EorQOC9YfN6diKBVXqdDfzWs58uVN8onul5lECY62RTCU9cjzADI+F1agGBceHMusf4vGjYLVqKvTzXKpuOWuAm5HLrrfklglcEySj7uLS8SMYGdq8Oukgmiz/Rd5p+RQI0WvEIYhFS3CZtF7UfFy6hfja2IwnDQ5hBrj4WQ+pHieN5lMtKk9BGY8ncespGoTVmEpCmeTzhkc86HEIjt+rbqwvPTgwYPecFCv15WA4WTcaDTQNZGoaTgXZj0kSeK7qQKVtBrjOKaoWzCeLtpZ8+eSNYMYF8xmn0LHONNbf1qkmSbIJUduESzqM49M0CarGPL0lyKiGAMkkjpW+gkRLyHE+2aFrFH/eRYWZDYf1AfSQCnGzoInGmRsEGsBQ/Go8sAAhHsK6RPmQE/+oCJFyYqUoJ6XTyXXkfHpkziCvHkRAmKtkkQ7juP4XpIkidZKJUU1d+M4EkIIKaSUCkCpRGutQbtyGtyJ0pRsdCThshoMZzbBSJkARBxq/HkQBCTGsjsAxFZKuHccJwwnjuMEQU1KOR6PQTmB5widTMZjoRMpBKgYtJagpIQkUQAHYoPzf9E4k9imGVQChBBBEAxHI9d1fd8Pw1AAVCoVBdoFZ2FhQUqJhaRLWj4LdM7gOJdWV1d3dnaUyWFtNptgss5IX+DrsKgdS3Ja0yZZXoZgNlD6Fj9YMERUXuNCp/XrEgQn/OL3W+bvJ0kcrehDiSQo6ifNi4WDOhOwBZmZ4oTaPQkb61dw2Go/MXFbrTV9p6KOUSQMyT8KL6MBp5dFtMJAI/yKEDP3xUks8X+z+wn+LkWwTonIQgjM9Sji/JNRduppPZL0pSQapdT8/AKGXfElcyoz8jjoWYDj5eXljz/+eDgcTiaTOI7b7fZwOPR9v1wtzSULFpGy+JLdWh46wYfCEBwZL2hdcRXScgmeuD8n6D/1h6tORVpnkTGBbE3UGtfISH0+1BVDKiGp6tQrzeJG8OZTXJZcIqZwSqaid4oA8VDKxSDBSqorkzuKN/i+jxoDqoQIzRh9BGzbQQMlWUIpzgJlDIo0YftoysjCMYlDnDK+4zxFyo5hkiSOCdfxfV+aYs2Li4t4FknWSXMG6dzDcRLHQaWysrKyt7c3Go2q1SpmYXJfM5iwR6VU4OafpoF3cobT6eo5luWO7lTpaMpHLLxJfxbBGccXSmO1BMZRUOa0mNIaEGtYjtUOYbo2tmOumtEuFWeqfHz42tNpRwJZM05XRcoK0Wn7GUF+sudSyR5Cdr4DIBlMialYKUKaQsPaZOtl3x2HiIcV0g1CiCSZRqeJdBRdkXZMtmzarZ6uFyRHVTeylv5F0eI4zly7NTc3l/35maVzD8c4JZcvX97Y2Hj48KHjOKPRqNFo4DljBAp8bZe0Q+vcMnTyHxJDZC+W0KE3iLRWW65dChZgkIXjx0GnpU0f5Ye4hpHQykRvJzKmeYsItkgk49QftxLecanwKFKwYfp0ESGr7BMscs0UIZtHGVqd4eCuTJi5YuE01hQU9QeHmqveQojjnihW/r7WFdLKsYdksqjVaktLS9QryBR9PIN07uEYAFSSVGu1lZWVfr+vtR6NRrVajXiITGzTm0ttx/wzV2Q43pH2x9mU/i0q1FnOBBbHl9wv0+mClhSxdLQSXey0mJJDz6MADbWQVY25FgZp/ShLlGsgmNLEn/KY7OxWrw4mNCO8y/tfRFlLLr0LDR0p/soUAwJTQohXI4E8OOaN85fyXBfyqodT0JilHYMJWBZsQ3OKgWXZCdUACMdxktDUB0EwNzfXqDeArWWdCd88a3Tu4VjiibaOs7a2tre3d//+fSHEeDxGl4Jg9U2mP0jKwtQhg4lZXUCbMB3CZWAYXdTPIgjg2gcH5aL7KXFApBPMLE/FEZXxo1NJf3iDVt+yVLR15T8HM3H8pbiCc5TeEr5wAcbDzC2J+4jEO5YShzJlNaann2D8LbGENJlMyF8tWPGWyWRCNQMgoz1YOrXIBIGRAYRXRuT3Ex9acIz2EBSKpJ6fur2iSHGhundzc3OdTod6Th4FvBndfafYpdOicw/HIITneVqpuVZrcXHx/v37tVqNLMVZbUUU2xwLmk9pbZwFidH556JNcTkMZdXwImSXLI8rq5byPw9F5FOhLKxg9447DlZvSbFCTAEm9srfK6sqosZkna1Xbrk6AfHEQs00TZmJwz3Zc8mlZkUE8kp4yhwkKKUMggBjFulgEQAIgsAaT75/t7CYHsffiO4scuWBiVnGEsNovC4KNzoBaWYjpouW5iulbDQatVpNs40jf8EzC8dPtBT04yCVRNJxtFJCSpUkN2/e/Oijj4IgqNVqyJeYd4Ah6K7rCpX/vtl0DzBwkHs/Fn6EdEjWCfpf4pI61v1FZCE1X06oR3AtUrIjJi0qEg/cLMj7jH55SKdf81MhaBnjnxTIRRiKoDAcDnN3D1zZ5LsEwXILyb/PQw6OSEXaq6VUAhMA3M9GbxephDopWYZFpVLhsRDSFHMYj8f8HTnU4tRg5DWNEqnAFixSiWS6ji2U8I9IO+ums6AhiiIMyRBCYDYHRpJR0Is2FbEpzAPM/oZSS7AKLhh9AlNI6GZrKUEx/1PpbRoxJabFo9FqUavVVlZWFhcXhRBREnuOf6T5Pht0/rVjIq2l46yvr+/v7z948IC4FndzQRDEcTyZTDxZJhUtJihBWM46XFwXLfui5X1cOi3xyfVoviqO28+EHe3KEYpLKcEc34QjlhrLu6S1Rj9etof0lKJx4N2g1kre69THM1dh5NfBWNgIVvAih0s+klnVFRvkcJYly5iGwkkdluSNn/mYO3JqCOb2kOxz+ZjzATl14iasKec40nVd13UTpTzPa7VazWbTGrTzQucejpVS0nGElFEYer6/uLR0ZTjsdruTyQQderhlq1Qq5GHIbee4Gq7Ki/MtgYnj0unCRJaKjB5F95eMm04X1SQU4EPK/+UwRPYfDuuaEX+uBbUl78XfSJVm353WOAPDXGVSwIGxhGaBCpKduWeNQ/btNLN+AvNSoDOtKGghFx/Lhy4XTy0ghrzjzbKN8xu4HHp0InFFzUqT5IVYPD8/j4HGAOCU6l5nkJ4FOAatcebjKHI9b319fTwev/POO5PJBA/IUkphzrTjOKKAMTjHHLpjAoYsFhgV2UyPu1l+3ETqEmGcLi04eSwfGjCYti5ylQ2YRZgHqFluIk70w6L+ZCG7/L1OiwgguBIqhBBgQ5JIh0VbEXI8kJG3b2kSNLBF/aFoM/p5brNE2chi7gIlB4lme51cONbMvkyNn67MIyMYyQnsYXNubn5+vl6vAxUsPMNBFLl07uEYj3p0XNf1vCgMQWvX865evbqzs7O7u9vv9xuNRrVanUwmYRiiLSyXyB53XDX5iFSyScy9Xq7IPDpx/YXguESBKrrOkYWWSvn9iEF0gBBpytZnbJArj9yV+rjh9bSIxDb/XDK5RdqxhdrSFPguaSdXXS1hbI6t9EGZGuJk+Ib0sQO5EfoWWJ/iaiLGoPRrpXWSJM1ms91uY4EEEu3nzi127uFYSAkGAgia/SD4/Oc//84773z88cfdbrfVaknKLCqYIoIkulLOQxbvHqpQH9eTW7RiT4uzLfDVpdFpJf3hbjT+gcLyIQ36kAZuAl/LJUjEK72Vo4n1W/70J4DdBEM8VkGnMyl4D8kPRvKGIiV4g3yHgdfJP2lHcGb6w8c5y97Z+xUrB0yvAGmjv2IHEZRox1kxf+KBtQg3u5o5Th1Hep63srJSq9elKWz/uLWZx0TnHo4BwCH7AFrlkgQAavX6tWvXlFIbGxvdbjcIAnT7uqLsAPbcz7kk09kEh+rUxzVWHFebPi7RsiEAFenaFxaViAf+1llFO7s4gb0df02OwvRDboM+CrCqTFUKmS7x/vgoF30sRZK/i0jbbWi4II8VKb4NTcY8z6WoM7kXj6IdW/3hJmORNnxbcGw1Lh6DdqzStQyllLV6vdVqdebnwRIk55DOPxxrDRjtyA6fR62gMz//2c9+Vkp5+/bt4XCIhZ2Elx/4wm2+WZUkS5zVOBYXacGnZZQ47v0l2pOlrlpK3BGJ75dz1SXrX+7O4p2hOFa8QoufjmHnvxKHnYmnmQWTYP24r3Ys4lohZwllKutD2gGF8ZfZQDe04XCxhP9iSSzF0r4p0jmXHHb2LpgwO82sSbn3ZwPdtE54ZEWR7slf/LFKPl78E8VSs9mkZGht6ikj2ziPOTP+1Oncxx0XES5XKeVoNPr4448/+OCDKIra7bbQgLGQSqnxeKxN8RRkd0iHXmmtExUdQHwatogvuZccLdQW0imlijzgOj6Id+ZwxuvoE6+XZ/2RCsZLC5W45kht5B94tzkclLjO6E5uuCjCZUvvo036464pQTodGQdE2qUGaclaEolhAT2Uigf+1hyqdEHAH5lErTranufhcXN8mpRSWCuZJhFD5XzfpyRmYEZ5nKwwDPE2/BNrBI/HY2yTzPrOtE7lVB2mlYKSgLtbefvcmuE4Dm5JlVIkZvC3URTRYal8xYGx29B1XK0Yp+w4TqQBs0uUUq7rLiwsrK6u4inRzwCdM+lxdKIVUqvVrl+/XqlUPvnkk4cPH863O3jUaaVSofOnh8Mhz1awhHz2AycLX6wVjiQfT50ETvRcYnE4kSot0mffnXq3eXacpXPl3n9aWi1WOKOHUg3JorSUIpuvyJwqW06FpYUyUcOEsJaMh4IcQuJVLkFJckN69jWzVvMSP6hIchWeYBcRFo8WJc1Ap43+kOExLnr5+5ZQ7lrD3BNuWJ/GoYupruM4TqvVWl5ertVqURSdYpWip0jPLBxLU8LKcZxKpXLt2jUASJJk0OsnSeL7fhRFk8kExW8QBLRsLM7gymlWh7V+Ank6YPYKp+NaB4qYm1tIc7XUbDv0gnz1Zvt5XEwvJwuVHtNTssQPnQJ2botKnyNFU4xlWoENCylrwOQW/3Cyjlk/1Cb0TbOACq4/WtNESEo6sswUv6YWgNmysed0pLfImI9QOY4LTnrOcjikRRrX/Y84DtZQg7Fuka6Nyjqqye12e3V1FetnPoEz954MPbNwDMZfT8cEXLx4cW5u7oc/eGM0GpGEp82yBRD0pyg4Ec7S2vBbmQnVpCVdaDs+pZeVLL+Zto3lqiW9NWmC2fvLZckJiApm0hZElaZpnBaRiKJHZ51+XLPLCjMLc7nYLlGWs7suehykMQhKtyNFSfxJusQ2V7HpjYghtbFlEwTzav25kpJiPzQ75s7KSbHeBYqFbpaIDazxkSauDhOgybTtOG6SJPV6fW1tjbD42VCN4RmGY8wBQfmPqlAQBEtLS6+//vqHH36Ih08HQYD35J41B1MJLzgca2Mp44HxxEbq+OXnjwvIRfxtwYo4LAiBo7ClU2cf8bhVV3j8aTLcWEGKGxoxc++35BCXxNmpPIG4onYsNZPzXvbm7ETodNRXtj/8RbTWQRCgWwJ5XqQtJ3xfpQ/zVUCeDOMvQjK+ZHysEaankymcGkdo9j2/1WqtrKwgFnPP3jNAzywco+faSNQDt1hnfv6VSqXT6dy7d+/Bgwe7u7v1er1er/NDaGitpvWhQiMyl+qWeM+9/3FQVluHUm1LsHA06wNf+bTIT+sV+DhD2tL9WImidLlkhTQC8tcntdGaUAuRLW03S0W2Y522cvABsVyOwErCc6/y9FdaEX6RimBBMN1MKie/Rwjhui461vhQECRDge3Y4hDridQ+H8NcspaP1W0KOEEbRRAEa2trnU5HKRWGIbr18NzhkkecF3oW3iGXiPPQWEHCVieqWqtdv3HjwoUL77333u3bt0ej0f7+PgVySkNTtkgH8UDGdmzpOFktpkivMV8f771Klj11TGVqI+Q/Oa/Pub0tgfXjUq4k04+/pgRukmiyyNJKy5jPo9aaRDifdEJDkTYcwxF2LRblDinHO5pKYLKTzFDUJaEEt/xSQBHfvXHJh+okOTO52OZvWsiu7L24CKGfq3Q+uoH0ozIhfSbrNjlFgiBotVpLa2uYeqeNoRxKFtd5o2cWjnEuqVorMZnjeSpJpJSVavXzr7566dKlDz/88KOPPqLCfZqREEJDTrqESNem4OJdsXoFwMx2j9s2StoTGF2G/iyiXMlhqYpw2rzO95X6CIFup+WlQWOFNvt6xxyEmqur6nRyNrBYMatZXWo4hmLbcVb7JvzNhWO6h6QsV7QJufhhNFbj9OIobLBCbBiG2cNJU1RgO85FWJ0hSIuEXModPRSf5H4UQjSbzfX19fnlFSzNiFYXPpjPAD2zcceFhK+b5hUA+OEPf7i9vb21tRXHca1Wq9fr6BxzvanbGndzaGuO4xjjlIljwYALxmzgqsC6f/gVJnCT8gLGFoZJ25z5gW3hszpL0Xw57OhJbgEEpshoUziNekXLWGbCSGmpK5Z6QI57+sArsQEDjuPy1WmBPgego4iTIsWNPEjKnPiJnv0iW3OR5ENMkVJiBC41RaEdPPsDI3ww4Mf3fTS4kXGAzw6Y6YtUvrjyfT8MQ1SBMUoE3draePPQ/YXd49tH/BN7GIZhpSBtiivR1DFhDNCofXueRwW8tNmRIHSidMTxH4/HGFwchqFSCgcKz7qsVqvY7UajcfHixfmlJSjOqn0G6JnVjg8lkd6pfeX113vd7qeffnrnzp3t7e3t7W0AkFJWa4EyFaQcxyFVAhuhLaE0iaRY5x4MU9JTcr2FQggnz5XHgZhfhKNVVitXRop+QsMCacWK95mjLZJVY8EakPNLXDs+sYwBU1YfzMCSuMXDw6hBHEyeAK1M+WMEcSwzL4wvBO+BYmGjMyHk+C85x/jPLX2WflLCRYLp8lyoE3FxzgPvrMHEK1T0B9mJzjHRWmOl3LW1tfmFBUgScM83X5XTLy4cIxFzTMbj5tzcy6+88vIrr2xvbX366ad3797d29vb29tDiV2v14UQlNGEajJKdYs1gZmbaa9Nh+IQcE/VnOLTSawFA4dFbpCSwi+WLNcs3GuWocdBhLR13hnN4nCfPSIs43rfCcQMDR3HQa4H8MGkvQuq5HSGUJIkQRDkaseHIiakWSgL4sS91m/pQSXjo9OHYIkCAuaTPPDKMI7luj/ejOsLAIIgWFhYWFxcBFYs7FmlX2g45vwaYMlqrZVS7XZ7bm7uM5/5jFLq7Xdu7u3t7ezs9Ho93KgGQVCpVEajkTbmac/z8PARYWosCOYfl+mjrImJp3/GB6uUE+kL1Fvk+yJbKt3JFa5DR4AWRvZ6VrHK1acssZHtxjklwZJE+CuXiLei69gUGgpQeKOdinKOObSRDV2Y6C5kFcrXgDQUgjxEO+ZCVDBvCjESGTSsVyuHY65bcOnCt4BkbXdddzKZ0JWEnUKNAam476QUcLTyKaVqtdry8vLCwsK0b+d8y3Uo/eLBsZ4aa3O/mhYKcF1S+b7whS8AwHg8RlDe2tra2NjY29tD0xv3CPEAIGJTqgBA6Q+QLuKl01lhrC92wAZXWnPvt9beofBh3cAbz8LxjJCOC8flTWXHGe3I2vg2EY4Rzgj1JAsyK3lqCRwjWwpznJ0Qgge6Ua+EEEWRP4ibPIuEHlHSnyyzUewz/VYI4bpuFEUYX7y0tORVKjqOIY7FMxHNVkLP+Osdg7QGLAGltWYRTkonnu/X6vVavb5+4QJoPZlMBoPBrVu3+v3+7u7u/v5+t9vVxieDH1zXJX0ZmyJNgeOgZIFWphcpTORbyEMX/LEwlG62IJ6LCn6nyEQlIz2rtmPS/h5RMnGIgfQ2CD11dNvBholhKPktirT1fMWCTYHOKxGVnUfeLLB5hII0Jc0MFPQT4nPqPG4FpmdzmEeQWQ/5X2uNTjwsUkFMtbKysra2JhwHtH7mgRjpF+Ilj0KKpahNWVkIAHCEBABt7HqO6waVSlCpzC8sAIBKEgxb3traevjwYa/XGwwG6JLGGihkWR6Px5Zzb+osKljglh6deyV7PxGBfvlbZ7VjrqRYF+kD0XnH3BLi4MUj5A4d0mw7BKnAfIO8HcIpYU4ulyZwmKSd7/uEYkRKqaJIAwte6QN6PiSdxmCCfETaJE2iugjuuXig++kzl/SaKfU0ILQERNrEgRer1er6+vrS0pJwXTjnVq9j0QyOp0TIIuTUGocuYceTCh0sJi8ew5aRTaXj1BuNeqOxfuECAIDWW1tbe3t7m5ubOzs7g8EgiiIePOSYc3BJBVBqekZRlixuRiqyHXMt9VhKnIXa/HGQWZwcrKk/HL6fGdsxD+az4OZY7VC9NLJoaa0dx0HjAH8Q4uN4PKZgSgDAqC+MfuPdwMaFEKrYeAVsNsEoHFj8jCQNuRlpxu2fH6YucIHN7cKOOcaF9oXUMvd4o3jgFm3XdavV6uXnnsO7p/JAqSSOHT8/6u6ZoV+8uOPHTIplfAwGA4zNGAwGW1tbw+FwMBhgNCieqUrhClxlxmhNzUx+wCyGrjN1BHFFFW/D7TBFtmKbUhzYrPmSQ3lA6EAKoE4iuoH/q80+VJt4LMEsFVkqKQ9P3eBunyKtX5mziCBtvSlXzLPvq0wZaMkS0pQp4k7KJiEFKY+S5SVrrSkWmPRcmj5yHgALJIi1HeNlKa0WSX2QxEwITht5wjL6nMDBvFCXkMF4bXtgEdD4ajwCndeNQ0IzguM4uqCimzQBQjiAB1znTotzUkGY8Xg8Ho8BAM13uHdEx12SJI7rU1g0hjBdvHjx8uXLJZP7DNNMOz5l4ju+arWK2y4AEEKMRqPd3d3d3V3E6P39/X6/jwxNNauUUmjoICWaA4rWWgpNS4UvQqpWzvURpZSl3WhmlBTp44KmuM9u5vBBPyccEWlPlEVFGvpxxb+FhtaYHJ1KNg103bKEctykDuRmD5a8VFFWXhHRydPW9SiKiENSurZKIF1FGkxtAJGOsSNRxHcwiTmZlJ9tCMwjHeuDQHsuPq3aPZwrhMkbxGgNIYTv+8PhENHWMeXk8edYdAJ5vt1ur62tzc/Pq8efxXo2aQbHj5eIq7DKyfLy8srKCm5FJ5PJZDL55JNPUInu9Xq4gUXtuFKpoBaMyg6pIQjHgsVO4Uqo1WpgMv3AFF6Rmeo8FtxwOJ4aKwuAjh6k0/vlQi3v+PG5Je2Q9ipMssCjJE9z7OAfeB8ISiz0Zz6uxxhzYg0vF7GkieO/0nWoNhtOOiIsBtIRjGpm++ZTSSTStiZ6a53evdGfCMRof6P7HceJVEK8ByznG23fnFu0OQsV1epGo7G2tra2tgbPUP3i49LMWHHKhE5kDkbEcNadiKdYOmAymQyHQ3IJdrvd0WiEZkfHcfDgkqlrHg4ObkASLEdDZsLsQKcqb4m08YH6pk3urCN09n7IAyBcgUVGhiI4tkodwRG0XY4pYEK5i57Lf8JbpmIOHGE589NF/JcCvyTLbuDtWL8FZqCARzBWQJIfMSaY202z4mrSs+vHI7xStzngivTJqnx8isRePAktYwg3qZOWgD9xXTfWCnVeqrWGmi/yuTbuRBQhWuswSoQQnU7n0qVL8/Pzwqj/JZP7DNNMOz5lIpWEiK9PSEcsgEEZz/Pm5+fn5+efe+45hOn9/f39/f3Nzc3t7e3BYDAajdArWK9VOEzQv1gTg7SkA8RhPmv+r2SBR0RCCK0PXP/8W0s5ord43CuH9uZ0ojsUh5eUkGY2a64dF5V1pzt5UR4OZNnGj9ulEuJSED+jsYKukLEiZlXkrVAzzQLjJEt7s4YCP5NLlqvP2TelXlHyNxjBj+04nst7QhUqUPMg5Z1ypgCg1Wqtr68vLCyA8Xz6z7rLrohmcHzKlIUwJAu2BKtlIdIRY4g1zWaz3W5fu3ZNaz0ej3d3dx8+fLi/v7+9tYnHSqFLGhMCcT1g1RUMeT5YkypHNQYWUWvtH7UJXSpCZI7jJTB03F1X+f2EKdaG97jEEScrjTRzn2JSMko42nFDWsm1Wst94rFtxxlXJ2Q0esLZJEm0AD4yXDWmx5HrEuMWaFOiWWWo7O5n2lTePkZrTaejApPKwmR4o+OU2zEUExtRFCGvJkmysLCEMW3A3NHPzNl3x6UZHJ8y8a0WZ24qB0NXhMmJslqgi7RC0i5B3et2Hz58uLW11e12u91uv98fj8fVahWxGOEYXYJJkvheJYvFkBcGO138xoVl/SuY9ZC6WoIvpwXHZPnRLDpVFBtJStq3sAb/5OPPlV9pCtdptvEHAPSynq4unO2nSNtqwTgD6OKB7dg7CI0gmxUxofV2fOvGP3CV2Xo11GfJFiFMbSPEU1IpyBVBhg6yb+AVFG9ofMOSh47jtFqtS5cutVotMFZmnrz6C0gzOD5l4m5r/GAte80ioiwNlGsoKlPGe3qbVnh8ybXr1wEgjiJ0A2LMxs7OTr/fJ2+e53na1dQsfxzek4VjkQeLmqV4UWv6pHaDYxEPcaN9ulNQb/dQ4jojZMzW/PN4PD4QUSzSAGtFlqjDFiXFp4Hk3q8S+71w8EkM6LSxQjgOD4yT5vhHMFFowEwHVERYsB0SwjRFYtBDsatSyISdHkLMyf90zLHcnudNwhBLg6J6i8YHRGE0uFWrVVQ40EaBcRTa2FjQg/psHO1xApq58s4dFcKQSpLJZILQvL29vbe3NxwOB/1pqSPXdXGdYAQrBT8RA0wXcxwK46PnaiC6KCmNFdIBT6iS45+8dFlK7864s/i/XDwA81wdpYIdARzphsJEVSPKIDRw7Z7/S9sRHtBNyIWBWWAMoMLsuEnmQenpmZM4yo3zFebsZ5GOL/akk+sSJOMsf3F0Fea6ZIk0KyZXYuifTCZo46KWtdZxHPuOi0qr4zhoIqO5JrMJcBOWI9HmEAQBSi9MgCZAH4/HYRg+99xz8/PznU4HxLNZEfBk9AsqhZ490kpJx6nWatVabXllJTFh9r3uoNvtbm1tbW1t9Xo9DP8UQtRqNWkq59K+Mo5jV6asikS4tgkUuCORL3jaNefCB6R3ANa/XOXkXx2XuMCg7Tk3dPJ/rUdwgwB1INsxq28iXUeYky+FMOFcKl1xglrg2jqph4jvuK+nqDWrtwSdYJRoxaKSIXOYCN6DcgUdD/Qnb5mraNb8SlN/2arjTL/yzCFYFDuB/RkOhwjTnudduHBhfX29UqnAL6pRoohmcPyMkLVQHdd1XBcAKkFteXn5xo0bSZIMh0OsS4dZglgLCZdWpVIJgsDzPKGna0kbx51jDvKBtM6L+EJ1cCwLgMoWuwEQzOZracpFittxd28EwTwvBlggF6T3BEiW8MjGWlnQc3TiLlOuzpML19J2hTtVkzHqwAqVEZmDmbldW6TPbeIzQm/B55H/yV8T2FZDsAgNGkZ0hACbxGzjkOYW1Nxxc3DhwoVKtQoAWinxixrTlkszOH5WiCMFgxvpTKfYlXKu1ZprtS5fuQIAw8FgMBhgObqeodFo5DmCKtLh2kbzX6VSUSbPGG8gowc3sHK8s/RHWpl8AVuoYellJxsJ2vtr44UjNOGN47MwEoAqLYAxWfACbLk/JEkDRgbw26g1jDvW7LwY/MCPlaGbUa8lkQBp/VcwIqxMlRQyRP2kZgXT0Em+KlMYy/d9Hs8HzCVAejEHZWAlt/graK3xvXjWKGrfKFE8zwvDsFKpJHFcYjn5haUZHD8rREAmMM92umySKKalS6qNUgpLhi4tLwOAVmo4HGI5uo37d8fjMZo1MNYVLc68PC7C9Hg8jqKoVqvpTPlQK6uKFCsL3bh2DBlF2AJri44L1mQrh7R2jJtujrBcteSYe4KHAgtk5IUyil4KACj5mIwVOF/c/M17otLV6wk0j7Lb4Cqwww755U+hvFCSECiGrdbogzIHxGjmM0Dcp9xooM3BM3qUzIlpBsfPFOGipD8FVnBOEy+ujyAupMS6dABw/cY1UGo0HHa7XTzLdWdnZzgckhbseR5Wh0EFGQ+dckwZfgvvssEhRdoxVwnhCAkmJchICAWZGAD+c7wB+0/gS8aZrPYH6Ri7rKsN0grsVASqqTHdMXXfqZIRHyiSATSGllUnW3QNe6vjg5M1+FBb4oT/SRUthLGWYJwvF6JcxUYZzKtWWbfxD9KcfcMHUJosyjiO6/U6AEyT9GbGijTN4PhZIdLsrOsqp2StZksdYArKU++8BJCy2mhUG42V9XVUbJRSDx48oMKho9FoMBhgC41GAw0asdl+4urVxgLL0VCz2GHImAKsvXa5lldEtF8m6BHpGo/0CM3ScHg3EDrH4zEXGyX6bDnJdIIGSSlynwLDYqWU77poHaJOCiH44UaCWSroCh9kak1kth0irfVbX3G9+EA+pY9QoLIYljgBJqtorgnulalIF4YhwrGQMokiIYQzQ2NGMzh+RsksfincaZ40w7Xpck0jndk5Kvy5uTpFsouXL1+8eBH1u+FwuLOzg4dUYZgzD64Cc8IQRU3RE0lLysIxrfkTAx8Rub+0Sd4l7RUycExwpvPMCJa0EGnbBf9cZDvmzeKDUFyhv44DK/6EdhsUXYc5F7lpRITIkImssNAQmJaNZdWUUjyNk2Mo/0CbAIJjxFyHFSblnSHbMXUMH4R7DjRZhJOJHwSO48xUY4tmccczeiRCTRlPEdzd3cWCzggKdKIrLlpcrp7jIiIQFgBAEARoXgRjmqTdruccqH4cjHjgFw/IK8ogECYSGQEa++P7/mQyARYbp1kgbW47iTlNQ7PMFCvbgscOkiJpmThoA3Ggh7ITF7NkmWtpKAaDQaVSwYtUt4QA1wJ9YNq6JQb4IEsTraxMfWRt3J4oMHAcKpWKMHUnwBTaRkNWEAQYuD2ZTEbhJI7jSqUynkyazebVq1fb7fZ0E+NIV/4iJkMX0Uw7ntEjke/71Wp1cXHxhRdeSJJkMBhg0vZ7770XhuF4PMbTqjCWzvd9zGrjFlIMrUVjojDZEBQA4MqpI4iMs7z6jKVfcxWVq8DA7Np8V26F5VI7x9JRRNqTZl0p+pWFqlm9u+RXwLRXKktPRIEi2CwdQa1MqgjpztaLU7O8b9l78CIiPjA5gYKQ5A2ZXKQ5EE9r7XkelgfC6zPt2KIZHM/okYgjoOM4c3Nzc3NzAPDcc8+FYYhVQylLcG9vzxFSmZJdGLOBy3U0GtG6RV8TYofQ08N7EEdIf/R9v8S4QVhs2SUgky4oGB3lfS3ctKwH1tNL2rFchdTbom5kWysCTcvaQBfpenbQuIDkP+RjlW2Kx0eT6k1f8X0GzVqlUqlixDFGIpcM0C8kzeB4RqdA5K0idJBSBkEQBEGn07l69SpWoZtMJjtb23gSyvb29v7+PlqZXddtNpuoAlMLiL8uq92MKjPaCmjBW7ZLghUi0vg4hgpW7+bEcCxYWIhMm+bp2yIFmXcb0jnfR78fjB7Ks12QRNoaLowxhCKsLbFkVZ7jA5XdPdCvaMr44PPxpz/Rl4D2DWDxcDPiNIPjGZ0CFVlalUlvQztyo9FY6Mxjyu94PO73+7u7u/fv3+/3+5ubm9oYKDE/EJe6707xhWozIozSnhdYgBo+NAsfwKphEHEYOi5xwWM9CA7Ti09GimU50lMs+afyMiF5944iciwriuU/pHYojJokARqXwjDkuxBgSnS90UDVmIImNehZljSnGRzP6BSIHHGSHTMq0mFqU/jDg+iFqFQqlWp1cWnp+eefByGGg8Hu7u69e/c2Nzex3D5Wn6lXAzRiYPEjHgYAaaMEj2+DzC6eoAEy0dD8RQ7VUrMaMd6vWGqy5T0raocr2pCRIhZZlgTqJ9rcaceA8o86wDVlEj9cO+adIQTnQ0f/8peiCBm6yH/LdWrNbNnNZhPhGACm3YDCk6p/MWkGxzM6BeKBveQyAhbmdaDhalY+ggXbYZbghYsXkziOogijNfb29rZ3HqI/cL/f01r7vl+r1bBaWGorjWiltBIHCCJ51LMjFUCsldYatAQALQVkvHaHwjHdRi1be3+6AUoR1orbpZ8Xada5XbKsCvyhFnzz26zPWcmh04EW2c4I40pVJpOb0BmdAWCSTUgRxoqalUoF8krLzghpBsczeiRKMocbcSAQJtqBgENTmlz6UFeN0Ky1MIrzwuIiaJ2oCKuGYvGj3d3dfr+/v78fhiFqyrxqKFqiSYPTpmYFZEr55KIP738RiYJYNO5p5CaUItuxyMRZHwrHuZKDolBIPvEK7rRpIJsyFVrjAsDC8XKsFMY0QX3GnD3BjlPQJv+QJyvWajXBSuPPsDhLs7jjGT1ZOi67CaVRv8bY3iTB2hoIzQ8fPuz3+7hnRyAIggBzKAjauOuPq6KIX65Ia9nYRxMQBqwIMraDtlGugUpTc1Iw4wCkvW1g4sDocxiGkNnaC1Ng04qDRnTD42vjOMbIX2Eya/KH2WxQrLrMWNxHm6J99I5WwUz6XIQPSZJgtDh2GJ9SrVaFEPPz81tbWzgp/X5fSzEcDq9fv/7ctWuYfY4dEEJoAAGzWLcDmmnHMzrzxDOYHafRbHq+v37hAgBgjdBer7e9vY1nCQ5HI8FShylCjkLoCEOn+b5xxO8nZMTTN6xAZlK3s/EblH7CtW+uHXN9U7OTxXN15PTbp4ywZP4+sPnkj5mgPlAjMlPtk3fAAmKrn1niNgf8jEJxPB4HQTAcDlF09YYD3/c7nQ6VkXMyx/vOCGkGxzM662QtXcdxsIwcLuxWqzU/P3/lyhX89t69e/1+f2dnB8+pwnL7YIwJ5BLMIoIwoXW05RfMA6bMWXBFpYdR70PCK4RW+CfHTc3ifCFTrCN7P6TPNgSmmJcYQ7jiLFlSH/8J/0yasvW5iBTLvaYARNd1h8MhHmxKh57gKenChNmVtPkLTjM4ntE5IJH2mEE66ZkMl1JKPOAVKYqi7e3tjY2N/f19NDePRqNeb+oSrFarvu97QnIoJK2ZcIrgBkEcDQWQlyBHNhBL67Tut37L3xHSBlxguio/D4krsyVqZu4T6TPvZO4TwZiAcxvHPB3quUwn4+ExTmEYCiFardbq6qprDq/iggFmOnKaZrbjGT1ZOia7aUgIcbg2yjfRVjKISpeSAAMxg8EAbRrb29u9Xm88HidJosIII53RMkuQR2ffcVMGXREsbI7wmiMjJSVDOtguq/xyAQN5LjvqA6nq+EGagj6FQ1dgC7a0chrPXFtzUftkv5amjB/VQZZSdrtdrfVkMtFaX3/h+UuXLvm+rzMpkdJxZrZjTjM4ntGTpWOym9L2sREWxtFFCw3JpolXMPeEbh4Oh4PBYDKZ3L11GzO5e70e2joxCYVqkmGwszBZfIiAZLcljRiMPZcgL2Enk1K3KfCAyw+OyBQ6xm8AZmSwLLZF61cU1GXm4gpYYAzVCbLgu8h2jEXmqAIGWoGwk0EQPHz4EJ8bBMHLn/tsq9WKokgDBEFAjc/gOEszY8WMzjRlIwcQLzDplr611FWZqcCg0xUq6vU6Ft69dPESKDUejegE7t3d3eFwOJlMyOhMpekAYDKZ8PAJepZIHw9IvcraNCCTVG3dk6sdk4GCQ2quKo1EMJprQebd1iy+AoxPkn8uap/6Qy+OvcJa2EmSzM3NXbhwodlsCild101YkPI0qW+WlZemGRzP6EyTnv6rAUCYFC4NGk85iZU54VhIABBSatAAAlU7pZXWWgop4OBUlESnIt5cLUCISr1eqdeXVlepfNzdu3f7/T5WPhqNRnQeSqvVIvsy33qLzInLFAc97XPaekvaKDdBgEFAjrOCBUUAyx+hApi545a9ny6KdFgxvQUJM2CCrch2nDWRaxP70ev16Ey8CxcvgiMBq2mns9WFEIlSUs4qVxzQzFgxoxnlEAFWHMe9Xg9DNQaDwf379+ksTqztS3jHq+RgydB6vT4ZDNHWQfom2TGUUrVaDQxCJUlSq9WiaMLNC2CQNwgCgjyuGvvu9BwpCr0gYaDYKaVY1xTDljHvBkwSHYamFcF6rBI0K5NxA3PttDkWWinlui4W2IuiKKg3sPzI1atXr127RlnRMzoKzeB4RjMqI65KA4BSam9vb2NjY2trq9vtYm0NOpjK9/1KpYJZaoCFk8SBQRmYrorHDHKHIf5ZrQZYABrVVTpcg4cMcziuBpVcOMbMDo7sZLElnVqZM5Mmk0mRjViLAzM9FwaI9UTY/zAM9/qDZrO5urq6vr7earXELBn6ODQzVsxoRkclRL2FhYWFhQUA0FrjAa+9Xu/hw4fj8Ri9gqQwJkkCrod4SpBKiIlKK5iwBCpZhweUYHoh5c5ZEcfUH/xAlmVgsIsmCDrtSUqJuS38Nm4nySXrVBG6s1KpUAwG2mRQimitFxcXr169Sor/DIuPTjM4ntGMckino4lpk44wRLnOrVar1WrhT+I4pjrO/X4fk7nH/QEqknheEQBgtAZGJghW0wPNHQCpk6R5ZAX1jQzWwIpVgomm4EEaXKGmpiSrSkwKb9E48EaIwFhdyJU3Ho8xW/qFF15YXV1FLM4eCzujcpoZK2Y0oxyyPGmH3ozGAbqilBoOh/1+f9TrY/EjtGzEcYwZKMLkrVCsmLEmV8gnBqwyETeY8A86UdxYQSYRCniQ5uA7VNIpvZAO/ZvWlfb93FdL9IEJm9tJgiAgK8poNBqNRtVqtdPpvPTZz1HsB+SVS55RCc3geEYzyiFuGdDpoAjI5G6AOdIUAUuyYhcqjPBXURRhbY2dnZ3hcLi1tSWlxLw1zEOp1+tSyn6/SzncqEeDCZrm2jrXmikgj7RjBEFUVzGEA80UQRDgoc4IweRjJL9clvjZS8CMG1gqE28Yj8e+71++fPnq1avCmx4LMEPhE9AMjmc0oxxCdZJn9FK9BUQxfjMqjLnBCUkUH2iI7FcbDx6EYbi5ubm3t9fr9UajUZIkw+Gw1W5C2poMxr5BKMxj7KQGgmNgySaW+44aCcPQ8zw0vGBYCAJuYbqHSrjJmB4dBMF4PMYgjXq9Pj8/v76+HlQqSVJWdWhG5TSD4xnNKIdyPWYAkD1zRJgKcFSTE5h6mEQx6qeIep7ngRAqSaheKAAkSYK25s3Nzf3uLhqdyYEWBEGz2aTSRdKUlJvisdJcI+YGCjTvYogbBrQppaIo8n0f4Rir+GNYSJEyG2vqiCKfJL7RYDDQWq+srDz33HPNuTkAiMIQhIPhIiUiakZFNIPjGc3ocdJxa3ToOAzDwWCwu7u7s7ODjkE8ZluYeqFYcR8R1hXSMjRLVlGTW8DxCpbtLynvSSYXDA4ZjEdc+0ZTjOd5Ozs7S0tLFy9eXFxc9IOAvcEMf09OMzie0YzOFKWyP5CUUh9//PFwOMRDuEejESq/juM06nWuuvKDUbgWT7BLgc8U34Z3uq5LmE5RGVLKSRhi+Bo2S/UuXnzxxVqt1mw2KXTa0AyOT04zOJ7RjM4QKZUyhliUJMl4PMZjULa2tnq93mQ8xq/Q3IwqM4+EAxNowYM0eGwcfUv5LNxOHRtzCia81Ov1y5cvX7hwoVarcRMH893N4PjkNIPjGc3oDJHWCfucqjQPrIYc3dPv9TAJZYrOkwkqsJiljWncdBoTuu+wETJZYINhGJKKTdFvAKABxuOx1rrT6Vy8eBFjioU5ulRkqo/O4PhRaAbHM5rRmaLCjAxOZG3gAWpJHE+DnUejjz/+GG3QGP9AdekwrAIA0KyBP0RzcBRFmFmH1gk8FlZIuby8fOHChXa7jVpzSeCEUkrKWWbZyWkGxzOa0ZmiKRxTpJpgR95BQUivZodDT++REgAm43G/38fCoXt7e3hUFZodqN48WicmkwmqxpVKpVartVqthYWFdrs9v7BARmfqQxiGRWkjM+34UWgGxzOa0ZmifO0Y00w4EBelb5DizAGUvur3+4PBAAvtU6Kz53mj0ch13Wq1Wq/XqcinEGI8HlPGx3A4FELwIm0UlcEeNIPjk9MMjmc0o8dIx11dGhLguX9wSG6bNlnLZMY9+CodPkGIiYXisuotRmhY+D6JQvqh60y/ipNpqDKvQI2fZ6d7PArN4HhGM5rRjM4EzUTZjGY0oxmdCZrB8YxmNKMZnQmawfGMZjSjGZ0JmsHxjGY0oxmdCZrB8YxmNKMZnQmawfGMZjSjGZ0JmsHxjGY0oxmdCZrB8YxmNKMZnQmawfGMZjSjGZ0JmsHxjGY0oxmdCZrB8YxmNKMZnQmawfGMZjSjGZ0JmsHxjGY0oxmdCZrB8YxmNKMZnQmawfGMZjSjGZ0JmsHxjGY0oxmdCZrB8YxmNKMZnQmawfGMZjSjGZ0JmsHxjGY0oxmdCZrB8YxmNKMZnQmawfGMZjSjGZ0JmsHxjGY0oxmdCZrB8YxmNKMZnQmawfGMZjSjGZ0JmsHxjGY0oxmdCfr/A2EjqTycvyzYAAAAAElFTkSuQmCC",
+ "text/plain": [
+ ""
]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "image_to_classify = next(iter(dataset[\"test\"]))[\"image_url\"]\n",
+ "scores = classifier(image_to_classify, candidate_labels=labels)\n",
+ "# show a sample\n",
+ "pp.pprint(scores[0])\n",
+ "Image.open(get(image_to_classify, stream=True).raw)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "vwkVzNhVNYnF"
+ },
+ "source": [
+ "Now we will test the accuracy and latency of a zero-shot image classifier on a sub-section of the `labelled` portion of the dataset."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 16,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 102,
+ "referenced_widgets": [
+ "a8ec0defdfe841caaf0caa4de07fb84a",
+ "14b5a9186b3a47108fafa070c06ab35c",
+ "3916f72e314d46e883db7e529210b098",
+ "34453701eb2f40df9a2a0154c02990d4",
+ "9e98837e01004eba948e9c2d3ea23e5f",
+ "f7d2ce646a4c42ed893383a4f75af2d6",
+ "2a967808803e44fea67bd9958542eb43",
+ "540bda296c904aa48c10db351a11ca66",
+ "c1f28cff3e064e46a8c9bdd0d705a099",
+ "fadb08eafb084a4e91f8406055beb502",
+ "8aae0197e477451a890b1f93cdeebe10"
+ ]
},
+ "id": "j_cJkBbbNYnF",
+ "outputId": "a228a4b3-4ba5-4014-8a6d-7166a688553a"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "id": "KNr0WEVWNYm-"
- },
- "outputs": [],
- "source": [
- "import argilla as rg"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " "
+ ]
},
{
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "J5ecm_MjNYm-"
- },
- "source": [
- "If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "CPU times: user 9min 20s, sys: 1.19 s, total: 9min 21s\n",
+ "Wall time: 2min 28s\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "fEDgKhSVNYm-"
- },
- "outputs": [],
- "source": [
- "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
- "# Replace api_key if you configured a custom API key\n",
- "rg.init(\n",
- " api_url=\"https://localhost:6900\",\n",
- " api_key=\"admin.apikey\"\n",
- ")"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\r"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "\n",
+ "\n",
+ "def classify_image(sample):\n",
+ " label = classifier(sample[\"image_url\"], candidate_labels=labels)[0][\"label\"]\n",
+ " sample[\"clip_zero_shot\"] = labels.index(label)\n",
+ " return sample\n",
+ "\n",
+ "\n",
+ "test_dataset = test_dataset.map(classify_image)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 17,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Accuracy: 0.8235294117647058\n"
+ ]
+ }
+ ],
+ "source": [
+ "zero_shot_image_accuracy = accuracy_score(\n",
+ " test_dataset[\"label\"], test_dataset[\"clip_zero_shot\"]\n",
+ ")\n",
+ "print(f\"Accuracy: {zero_shot_image_accuracy}\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "sgnz7XtXNYnG"
+ },
+ "source": [
+ "😞 Zero-shot image classification with a CLIP model gives an accuracy of **0.82** in just under **2 minutes** for only **20%** of the test data. This score is not impressive. Let's see if the text is more reliable."
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "OEV0FJUmNYnG"
+ },
+ "source": [
+ "### 📚 Text\n",
+ "The product description and name also contain valuable information. Let's see what zero-shot classification of those can achieve. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 18,
+ "metadata": {
+ "id": "edJSqNnaNYnG"
+ },
+ "outputs": [
+ {
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Downloading (…)lve/main/config.json: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.15k/1.15k [00:00<00:00, 711kB/s]\n",
+ "Downloading pytorch_model.bin: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.63G/1.63G [00:06<00:00, 243MB/s]\n",
+ "Downloading (…)okenizer_config.json: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 26.0/26.0 [00:00<00:00, 16.5kB/s]\n",
+ "Downloading (…)olve/main/vocab.json: 100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 899k/899k [00:02<00:00, 401kB/s]\n",
+ "Downloading (…)olve/main/merges.txt: 100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 456k/456k [00:00<00:00, 1.40MB/s]\n",
+ "Downloading (…)/main/tokenizer.json: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1.36M/1.36M [00:00<00:00, 2.81MB/s]\n"
+ ]
+ }
+ ],
+ "source": [
+ "classifier = pipeline(model=\"facebook/bart-large-mnli\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 19,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 102,
+ "referenced_widgets": [
+ "0ad0c441b68a48c0a8fb9436050d2a1d",
+ "1183c04b5df54255bad175f72d4a9153",
+ "4378864bd5004edcba65b59fecbe84d3",
+ "6f63b713ac2b4358b939c2818d15e67f",
+ "89ef6f10348a408a8546452e1fa74528",
+ "7451e0901e6f40f5971359c9ac5d47bb",
+ "fe63dbf326194c6c813327bd643a85cd",
+ "5d9977f50b704bd5b4a371387cc71a17",
+ "c7d383a298da44e491c81887632a89ae",
+ "4c46328ead39408995ba5a9837d57e94",
+ "35a4c4fa675546ccb34d38b6e1c7139c"
+ ]
},
+ "id": "aMK540c0NYnH",
+ "outputId": "5b1b7988-fa18-4945-9244-de8ab900325d"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ " "
+ ]
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# # Set the HF_TOKEN environment variable\n",
- "# import os\n",
- "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
- "\n",
- "# # Replace api_url with the url to your HF Spaces URL\n",
- "# # Replace api_key if you configured a custom API key\n",
- "# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
- "# api_key=\"admin.apikey\",\n",
- "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
- "# )"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Accuracy: 0.8235294117647058\n",
+ "CPU times: user 5min 41s, sys: 1.29 s, total: 5min 42s\n",
+ "Wall time: 1min 33s\n"
+ ]
},
{
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "VgNib6EDNYm-"
- },
- "source": [
- "Finally, let's include the imports we need:"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\r"
+ ]
+ }
+ ],
+ "source": [
+ "%%time\n",
+ "\n",
+ "\n",
+ "def classify_text(sample):\n",
+ " label = classifier(sample[\"page_description\"], candidate_labels=labels)[\"labels\"][0]\n",
+ " sample[\"bart_zero_shot\"] = labels.index(label)\n",
+ " return sample\n",
+ "\n",
+ "\n",
+ "test_dataset = test_dataset.map(classify_text)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Accuracy: 0.8235294117647058\n"
+ ]
+ }
+ ],
+ "source": [
+ "zero_shot_text_accuracy = accuracy_score(\n",
+ " test_dataset[\"label\"], test_dataset[\"clip_zero_shot\"]\n",
+ ")\n",
+ "print(f\"Accuracy: {zero_shot_text_accuracy}\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "IhAmyXNGNYnH"
+ },
+ "source": [
+ "😞 Text classification takes less time, but accuracy is also less at __.79__. This shows that some information is held in images that are not in the text. It would be great if we could consolidate this information. 🤞\n",
+ "\n",
+ "Furthermore, both of these approaches use large language models that consume a significant amount of computation."
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "s_ecJF0ZNYnI"
+ },
+ "source": [
+ "## Consolidate data labeling\n",
+ "\n",
+ "The scores above from two zero-shot classification approaches reveal that the task is possible but challenging using a zero-shot approach. \n",
+ "\n",
+ "With (our modified) Argilla, we can re-label the dataset and combine the information from image and text. Then, we can perform few-shot learning on the dataset.\n",
+ "\n",
+ "Spoiler: this should give us a better score than the zero-shot approaches, by combining information in image and text. Furthermore, our resulting language model should have lower latency than the zero-shot models."
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Bulk Labeling with embeddings\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### 📷 Images\n",
+ "\n",
+ "Now we can use a clip model to get image embeddings for the images in the dataset. We can then repeat the process of adding vectors to our dataset, but now with an `image_vectors` key."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load CLIP model for image embedding\n",
+ "image_encoder = SentenceTransformer(\"clip-ViT-B-32\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def encode_image(image_url):\n",
+ " # utility function to encode image\n",
+ " image = Image.open(get(image_url, stream=True).raw)\n",
+ " vector = image_encoder.encode(image).tolist()\n",
+ " return vector\n",
+ "\n",
+ "\n",
+ "# Encode text field using batched computation\n",
+ "dataset = dataset.map(\n",
+ " lambda sample: {\"image_vectors\": encode_image(sample[\"image_url\"])}\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "2pkY0NZ5NYnI"
+ },
+ "source": [
+ "### 📚 Text\n",
+ "\n",
+ "With Argilla, we can annotate samples using semantic search and the 'find similar' button. There's a complete tutorial on this [here](labelling-textclassification-sentence-transformers-semantic.ipynb). It requires the recently added Similarity search features.\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {
+ "id": "w0JW4z8nNYnI"
+ },
+ "outputs": [],
+ "source": [
+ "# Define sentence transformers model for text embedding\n",
+ "dataset = load_dataset(ELECTRONICS_DATASET, streaming=True, split=\"unlabelled\")\n",
+ "encoder = SentenceTransformer(\"all-MiniLM-L6-v2\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "metadata": {
+ "id": "BmZPHluTNYnJ"
+ },
+ "outputs": [],
+ "source": [
+ "# Encode text field using batched computation\n",
+ "dataset = dataset.map(\n",
+ " lambda batch: {\"text_vectors\": encoder.encode(batch[\"page_name\"]).tolist()},\n",
+ " batch_size=32,\n",
+ " batched=True,\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Upload to Argilla\n",
+ "\n",
+ "We can upload multiple vectors to Argilla. We just need to use separate keys. We will use `image_vectors` and `text_vectors`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Turn vectors into a dictionary\n",
+ "dataset = dataset.map(\n",
+ " lambda r: {\"vectors\": {\"image\": r[\"image_vectors\"], \"text\": r[\"text_vectors\"]}},\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 84,
+ "referenced_widgets": [
+ "a07c719a0ffc41f9a0e8227562dc69d1",
+ "dc03df51df3845edac3492bc3ee3391e",
+ "ebf96889240d44f0b5a16fd945090213",
+ "c011f7d4aa7b4f66997d1c88804afacf",
+ "dcadb8f0088d43f6bc1a2cdc72cff730",
+ "b73e9fcc9e634c2780cc880cc65cc678",
+ "57ed9e91eb2f45a18ee08f6989bec0a4",
+ "4ae8401caf984baf9fc132490f11646a",
+ "e145a20bac85423a8ac6a1841f7ff4d7",
+ "0b4387ee1a0d4f769b391dcbc4d5477c",
+ "b8da2402e8e245d2b5ff592f602f138e"
+ ]
+ },
+ "id": "gaEQm1BdNYnK",
+ "outputId": "0dfb1340-a685-4071-a4a1-71e6aa42463a"
+ },
+ "outputs": [],
+ "source": [
+ "# we need to set the metadata field length to 200 for longer urls\n",
+ "os.environ[\"ARGILLA_METADATA_FIELD_LENGTH\"] = \"200\"\n",
+ "\n",
+ "# instantiate Argilla records with vectors\n",
+ "records = [\n",
+ " rg.TextClassificationRecord(\n",
+ " text=sample[\"page_name\"],\n",
+ " metadata=dict(_image_url=sample[\"image_url\"]),\n",
+ " vectors=sample[\"vectors\"],\n",
+ " )\n",
+ " for sample in dataset\n",
+ "]\n",
+ "dataset_rg = rg.DatasetForTextClassification(records)\n",
+ "\n",
+ "# upload recors with vectors to Argilla\n",
+ "rg.log(\n",
+ " records=dataset_rg,\n",
+ " name=\"electronics_with_vectors\",\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "I5G3haEnNYnL"
+ },
+ "source": [
+ ""
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "-x1c0NC4NYnO"
+ },
+ "source": [
+ "## Fewshot Classification\n",
+ "\n",
+ "We can now use our newly labelled dataset to train a classifier. We will use a SetFit model due to the limited sample count. Notice the significantly reduced inference time and increased accuracy. \n",
+ "\n",
+ "A complete tutorial on few-shot classification with SetFit and Argilla can be found [here](labelling-textclassification-setfit-zeroshot.ipynb)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "M3rtb_KHNYnO",
+ "outputId": "4858c080-44e4-4d84-b22b-b01b3ce28b0f"
+ },
+ "outputs": [],
+ "source": [
+ "# load the 'newly' labelled dataset\n",
+ "dataset_rg = rg.load(\"electronics_with_vectors\")\n",
+ "labelled_dataset = dataset_rg.prepare_for_training(framework=\"transformers\")\n",
+ "# # To try the prelabelled slice from HF Hub\n",
+ "# labelled_dataset = load_dataset(ELECTRONICS_DATASET, split=\"labelled\")\n",
+ "# # To evaluate on the larger test set\n",
+ "# test_dataset = datasets.load_dataset(ELECTRONICS_DATASET, split=\"test\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "xwTr038lNYnO",
+ "outputId": "70f363ab-78ec-4e3e-c009-fd266968ca30"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import os\n",
- "import pprint as pp\n",
- "from requests import get\n",
- "\n",
- "from datasets import load_dataset\n",
- "from PIL import Image\n",
- "from sklearn.metrics import accuracy_score\n",
- "from sentence_transformers import SentenceTransformer\n",
- "from transformers import pipeline\n",
- "from sentence_transformers.losses import CosineSimilarityLoss\n",
- "from setfit import SetFitModel, SetFitTrainer\n",
- "from PIL import Image"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference.\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Load SetFit model from Hub\n",
+ "model = SetFitModel.from_pretrained(\"sentence-transformers/paraphrase-mpnet-base-v2\")\n",
+ "\n",
+ "# Create trainer\n",
+ "trainer = SetFitTrainer(\n",
+ " model=model,\n",
+ " train_dataset=labelled_dataset,\n",
+ " eval_dataset=test_dataset,\n",
+ " loss_class=CosineSimilarityLoss,\n",
+ " batch_size=16,\n",
+ " num_iterations=10,\n",
+ " column_mapping={\"page_name\": \"text\", \"label\": \"label\"},\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Yr7j8UldNYnP"
+ },
+ "source": [
+ "Now let's train ✈"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 240,
+ "referenced_widgets": [
+ "76252be98eed408a8f08fab0edca8d86",
+ "5c634542ed3940a1a8ac3f73b634e88d",
+ "b0721a5abbc044a59a9d119c65523ec3",
+ "822cf5f25bbb4845a8df5b09df0c88f5",
+ "d24e98b7ba64420588aeee6c0a7d787a",
+ "de16a90176c24ff8aa76eb0af1281d22",
+ "c9464f1ec9e64361b0e621c664514ada",
+ "74cce0696fd74f999b8476974ec74f30",
+ "cad144a0ae5948f5896d12f83c044dfd",
+ "e831d4f1ad4d4f5f97bc9569a5294ca9",
+ "71f5c4292d4f4564bb0bf5af9368163f",
+ "3eb4c6ae76fd4118be2e466e560f6746",
+ "993a0a13bc7a4deea5cded6df4e27fd1",
+ "6d0477f7d5744f55a7882de18b923101",
+ "9b506c9457f046d4a91395935a934790",
+ "429a122883f6429d90c994558fc50cfc",
+ "9411b438e2634a3ebd0d1617d2ec3a7b",
+ "ffbfe6debabc4f5c8cad8383d596946e",
+ "e94edd188d8942e6b7f01cb1b962ba46",
+ "7059d83f776f49c887879e870bb71ee8",
+ "a696789fe05c40b281a0090625e66004",
+ "02b23c74c75e425cafe0c231b2892c47"
+ ]
},
+ "id": "qqp69mUVNYnP",
+ "outputId": "dc1d2b88-a61d-4f78-fd98-c8c5510b7972"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Enable Telemetry\n",
- "\n",
- "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Applying column mapping to training dataset\n",
+ "***** Running training *****\n",
+ " Num examples = 5040\n",
+ " Num epochs = 1\n",
+ " Total optimization steps = 315\n",
+ " Total train batch size = 16\n",
+ "Iteration: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 315/315 [00:53<00:00, 5.94it/s]\n",
+ "Epoch: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 1/1 [00:53<00:00, 53.04s/it]\n",
+ "Applying column mapping to evaluation dataset\n",
+ "***** Running evaluation *****\n",
+ "Downloading builder script: 100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 4.20k/4.20k [00:00<00:00, 4.10MB/s]"
+ ]
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "try:\n",
- " from argilla.utils.telemetry import tutorial_running\n",
- " tutorial_running()\n",
- "except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "{'accuracy': 0.9117647058823529}\n"
+ ]
},
{
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "izZIHEzYNYnB"
- },
- "source": [
- "## A 'real-world' multimodal dataset\n",
- "\n",
- "The dataset samples contain a `page_name`, `page_descriptions`, and a `label`. The dataset is split into two parts: `labelled` and `unlabelled`. The labelled portion is the result of my annotation so we can test methods. In reality, let's say this doesn't exist 😏. "
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "trainer.train()\n",
+ "metrics = trainer.evaluate()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "0.9117647058823529\n"
+ ]
+ }
+ ],
+ "source": [
+ "fewshot_relabelled_text_accuracy = metrics[\"accuracy\"]\n",
+ "pp.pprint(fewshot_relabelled_text_accuracy)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "TLEiVC_KNYnP"
+ },
+ "source": [
+ "## Summary\n",
+ "\n",
+ "In this tutorial, we have learned to bulk-label a multi-modal dataset using a modified version of Argilla. We compared a few-shot classifier trained on the bulk-labelled dataset with zero-shot classifiers of image and text. The results show that the few-shot classifier is able to achieve a higher accuracy than the zero-shot classifiers. Furthermore, the SetFit model is significantly faster than the zero-shot classifiers.\n",
+ "\n",
+ "This approach can be applied to classification tasks with limited data and can be used to train a classifier with minimal human effort.\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ ""
]
+ },
+ "execution_count": 54,
+ "metadata": {},
+ "output_type": "execute_result"
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 438,
- "referenced_widgets": [
- "d2cf4d1133a2421aa1ec980bdca4fecd",
- "bcfc02b30e024b5c81856bcd9e191d07",
- "eaefc3e261a7453cb66905079b71f16a",
- "ac546920996e4ca081e438999ae19fc8",
- "c54e39be37434e0d826d86041ef1de96",
- "f4d6914361f44912b0d7ca1b16496606",
- "b3c787a117ae42738c315c5cdabd508c",
- "edf41d1718984e7dbef2b65799935fd8",
- "20dad493a55343efaabbc6f7c1966cf0",
- "b91ae0e5a43143e68f53099e85f1afc2",
- "a1ec5941f87f4b23bc72188313ac9275",
- "6482c21eb5d449f795a9e39fcae6a46e",
- "2db4287290eb4c25bc440fc3f2f1d258",
- "b470309f2a6b4878852ae9e589764edf",
- "63de802f18e44b16a3a2f5029ad7b705",
- "670f91d2101c4d87997c228210646f89",
- "d8f11947b0d048489bdd649479c50b1f",
- "70eee61c4d3842f69569dee3ac8773fa",
- "26175752e18140b4a944bfc55f77a069",
- "c13ea0c47f6e48f0a804fa9f2f5ce544",
- "9e17b92da1aa43d0aa980db275437fde",
- "d2186c7599164e528aa3d92f6999a9bb",
- "35f1224563044be39e254342c6c5b635",
- "87a472fb7193452faea139ab6bf1c9be",
- "eb9b58dab8044074b1cac3e39f26e561",
- "6359430089654ca4bdd587c125b317a0",
- "be06bfcbe41040369d025626b8227b6b",
- "082dfef575c94327a5377c3f7278cf5a",
- "84d1a26c298b42cd820bcd2e4302a443",
- "0987a64b9cea4850998e78d76d31c3b8",
- "fa0573f2ebd84a5ba5ebdc0452362887",
- "e8e295d7292245fb9f17413599d59960",
- "0286aff9c56b43e7abd15f17dd33d6db",
- "8b161e12449f4078bb64245518bb596d",
- "71181e8f913b412fb72917c55a151723",
- "dcd3ba169df34a0395eb5d03405f8835",
- "d0b05902a4db4abd8c10cab6178a59ec",
- "8294163893bc40c2bef26032babb3938",
- "735a35f35dee4d548cd6284f9e3e1f5a",
- "b7dfa3f0d2374482af6dc25ec8dd65df",
- "c65f5b1c90e24d47a060c0b362749b82",
- "01bc8315e7cd4064a8653eae44cea192",
- "36156c14c8d544deab0ea65ac5ba7508",
- "15d85f4fe15545bc85d3717d7af7bbb1",
- "f8d66c452a6d4347b9a8247ad135fd35",
- "3c8aa1d02f1541bdb59b0a8dff27148f",
- "88677a526eae40a1b5e0b5e9e7a25218",
- "eaf5ad9e9e3240799128ef8bbf0a195a",
- "48f63d3af7604e8abe4ca94dbae63f48",
- "27d3f1981c244549901253b3266de1ff",
- "9aae9636296a4245989388403103e0eb",
- "139f9e8c84f447b985c2080c17d14a8c",
- "1d37c4be099b44b5bc67c2cd976693b9",
- "1ad0c7637293497a9a75bfa45c1a06e8",
- "4fc202bfa55842b48e59d4a800026331",
- "bbb9442a3ec1463790bcf6533f3be905",
- "16bd920f956f43b3921b79cec4d76d87",
- "c12bc29016124339a6c3cb11f21bfaf0",
- "a6b61668ad1146e78c2a9b2556b23703",
- "14cc3d3e93c54226bc2f38a4745c1d23",
- "16fccf90f2974a938167677b189b67a2",
- "714590bcc27e4efc837f870812929f3e",
- "9cef8cafa31a47fd9f7410478b5415e8",
- "2e4deb55d7b74e2fa09c9f925fd2e751",
- "2433cd897d734c78b3d4a62602865415",
- "9fac34a2d3e44c95a2591b26ddae5c1e",
- "88edaefb70c34854ad6f16d5a82b40e7",
- "ba93d6edfed5434397d869ba2bd7aca5",
- "7282d9ef9a754bcfb1297081296aed5a",
- "946c67b0a3f54ce09c35db7bef44991c",
- "f41cf420defb4b109f836fecea8e1d15",
- "c10b310f7f6344b39c8d165c887d28f1",
- "052b62b792fc4f1183ffb91f18d655c7",
- "c4a54f5262014b77964d42429d5e43bf",
- "cf540757bc79428bab170e157eb9381f",
- "03390d37eba24242ae6c2099d5fe7863",
- "88901ee5ca86458288bf82a22db7e379",
- "498becb6513a44a4a17cff76a8ae4666",
- "5066a8fd18d94f9da0aa0cf354d13452",
- "a8d41a62f1b94575a744c12a399de8c8",
- "e8cc8dbe0d164a9eaec472ffe1351db9",
- "0d3cbcf01ca14063be3ae129db78fdcb",
- "1a951db095d441e295bb9ff00d4dba30",
- "e120b072886b4d4f94ef506ca3d6a605",
- "d5fea5e89ebd4d298c169526eadeeb32",
- "8513e53c07c04d75999be1fec6d5b491",
- "e7da6c65cec140c3801fca68bcfdebcc",
- "eccef3d9147242d9a1e42f4abd0ebe4c",
- "b77605d5afb94b8cb579f3c690ea4203",
- "aedab95f1bb144c38180f2b2e71c14ff",
- "49d11634ca864c04bc87e472d3563e73",
- "20163aeba069412eabd3b506a339667f",
- "feef17ea90ae4105b2ffb56841ed2adb",
- "e97c5c6e969842468f53520dfba1ee8f",
- "89b04536243a48049eb777f00fe49555",
- "94502dbbc1ab42f48d2d375a8265dbb4",
- "090c3e60ff0d4e5fa930767672802d86",
- "b577441dde9e43cb9644244ca0cf336e",
- "b850f11900ae4874963b35e0b86470e5",
- "963fbb3d5b9f4848a8cadf2a2d2b264b",
- "52ad436eea8649058457d6380f618ea2",
- "5817239aa98046e9be3736688f8c1a5f",
- "c48e02dc87d74d50ba6de4aefeb1441e",
- "ecc225aab6804d97b1cdd383621c9695",
- "fcfda905fd4448b3940b04f4136c53a4",
- "16162c715d4745f3aa91c7f38a78a32d",
- "4c7287d821f1471e9ebdafa2f8229d28",
- "5a65206e09c04176bfd33bcb31cd1ad5",
- "9ac85729093c41c1b2a0fef5191b8ab5",
- "d774f350171b4c3fb6be768dbf11c726"
- ]
- },
- "id": "gdVdxJt-NYnC",
- "outputId": "c54aa613-806e-42fa-b4de-f374c78b1d50"
- },
- "outputs": [],
- "source": [
- "ELECTRONICS_DATASET = \"burtenshaw/electronics\"\n",
- "dataset = load_dataset(ELECTRONICS_DATASET)\n",
- "labels = dataset[\"labelled\"].features[\"label\"].names\n",
- "int2str = dataset[\"labelled\"].features[\"label\"].int2str"
+ "data": {
+ "image/png": "iVBORw0KGgoAAAANSUhEUgAAAiMAAAJ6CAYAAAD6q0KOAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAABBdUlEQVR4nO3de1yW9f3H8TcoooaCZuCJRFHzhGZgDs8HPKdZblmZkqlzLQ9J9ZuWQlppueXMrEzNpi1TOznLpiWGR0pTYZZpmiCGouIBFAyUm98frXtjYJOWfPS+Xs/H435Mvvd1y9vHvuHb7/W9rsursLCwUAAAAEa8rQMAAABno4wAAABTlBEAAGCKMgIAAExRRgAAgCnKCAAAMEUZAQAApigjAADAVHnrAJfD5XLpyJEjqlKliry8vKzjAACAy1BYWKizZ8+qdu3a8va+9PrHNVFGjhw5ouDgYOsYAADgZzh8+LDq1q17yfeviTJSpUoVST/8YapWrWqcBgAAXI7s7GwFBwe7/x6/lGuijPx4aqZq1aqUEQAArjH/bYsFG1gBAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU+WtAwAAnCVk4mrrCB4h9dl+1hF+MayMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU+WtA3iSkImrrSN4jNRn+1lH8AjMyV8OcxK4clgZAQAApigjAADAFGUEAACYoowAAABTP6uMvPTSSwoJCVHFihXVtm1bbdu27SePnz17tm666SZVqlRJwcHBmjBhgr7//vufFRgAAHiWUpeR5cuXKyYmRnFxcdq5c6datWqlXr166fjx4yUev3TpUk2cOFFxcXH6+uuv9dprr2n58uV6/PHH/+fwAADg2lfqMjJr1iyNGjVKw4cPV7NmzTRv3jxVrlxZixYtKvH4rVu3qn379rr33nsVEhKinj176p577vmvqykAAMAZSlVG8vPztWPHDkVFRf3rN/D2VlRUlBITE0v8TLt27bRjxw53+Th48KA++ugj9e3b95LfJy8vT9nZ2UVeAADAM5XqpmeZmZkqKChQUFBQkfGgoCDt3bu3xM/ce++9yszMVIcOHVRYWKiLFy/qd7/73U+eppkxY4amTp1ammgAAOAadcWvpklISND06dP18ssva+fOnXrvvfe0evVqPfXUU5f8zKRJk5SVleV+HT58+ErHBAAARkq1MlKjRg2VK1dOx44dKzJ+7Ngx1axZs8TPTJkyRUOHDtXIkSMlSWFhYcrJydFvf/tbPfHEE/L2Lt6HfH195evrW5poAADgGlWqlZEKFSooPDxc8fHx7jGXy6X4+HhFRkaW+Jnc3NxihaNcuXKSpMLCwtLmBQAAHqbUD8qLiYlRdHS0IiIidOutt2r27NnKycnR8OHDJUnDhg1TnTp1NGPGDElS//79NWvWLLVu3Vpt27bVgQMHNGXKFPXv399dSgAAgHOVuowMHjxYJ06cUGxsrDIyMnTzzTdrzZo17k2taWlpRVZCJk+eLC8vL02ePFnp6em64YYb1L9/fz3zzDO/3J8CAABcs0pdRiRpzJgxGjNmTInvJSQkFP0G5csrLi5OcXFxP+dbAQAAD8ezaQAAgCnKCAAAMEUZAQAApigjAADAFGUEAACYoowAAABTlBEAAGCKMgIAAExRRgAAgCnKCAAAMEUZAQAApigjAADAFGUEAACYoowAAABTlBEAAGCKMgIAAExRRgAAgCnKCAAAMEUZAQAApigjAADAFGUEAACYoowAAABTlBEAAGCKMgIAAExRRgAAgCnKCAAAMEUZAQAApigjAADAFGUEAACYoowAAABTlBEAAGCKMgIAAExRRgAAgCnKCAAAMEUZAQAApigjAADAFGUEAACYoowAAABTlBEAAGCKMgIAAExRRgAAgCnKCAAAMEUZAQAApigjAADAFGUEAACYoowAAABTlBEAAGCKMgIAAExRRgAAgCnKCAAAMEUZAQAApigjAADAFGUEAACYoowAAABTlBEAAGCKMgIAAExRRgAAgCnKCAAAMEUZAQAApigjAADAFGUEAACYoowAAABTlBEAAGCKMgIAAExRRgAAgCnKCAAAMPWzyshLL72kkJAQVaxYUW3bttW2bdt+8vgzZ87ooYceUq1ateTr66vGjRvro48++lmBAQCAZylf2g8sX75cMTExmjdvntq2bavZs2erV69e2rdvnwIDA4sdn5+frx49eigwMFDvvPOO6tSpo0OHDikgIOCXyA8AAK5xpS4js2bN0qhRozR8+HBJ0rx587R69WotWrRIEydOLHb8okWLdOrUKW3dulU+Pj6SpJCQkP8tNQAA8BilOk2Tn5+vHTt2KCoq6l+/gbe3oqKilJiYWOJnVq1apcjISD300EMKCgpSixYtNH36dBUUFPxvyQEAgEco1cpIZmamCgoKFBQUVGQ8KChIe/fuLfEzBw8e1Pr16zVkyBB99NFHOnDggH7/+9/rwoULiouLK/EzeXl5ysvLc3+dnZ1dmpgAAOAacsWvpnG5XAoMDNT8+fMVHh6uwYMH64knntC8efMu+ZkZM2bI39/f/QoODr7SMQEAgJFSlZEaNWqoXLlyOnbsWJHxY8eOqWbNmiV+platWmrcuLHKlSvnHmvatKkyMjKUn59f4mcmTZqkrKws9+vw4cOliQkAAK4hpSojFSpUUHh4uOLj491jLpdL8fHxioyMLPEz7du314EDB+Ryudxj33zzjWrVqqUKFSqU+BlfX19VrVq1yAsAAHimUp+miYmJ0YIFC7R48WJ9/fXXevDBB5WTk+O+umbYsGGaNGmS+/gHH3xQp06d0vjx4/XNN99o9erVmj59uh566KFf7k8BAACuWaW+tHfw4ME6ceKEYmNjlZGRoZtvvllr1qxxb2pNS0uTt/e/Ok5wcLDWrl2rCRMmqGXLlqpTp47Gjx+vP/zhD7/cnwIAAFyzSl1GJGnMmDEaM2ZMie8lJCQUG4uMjNRnn332c74VAADwcDybBgAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABg6meVkZdeekkhISGqWLGi2rZtq23btl3W55YtWyYvLy8NHDjw53xbAADggUpdRpYvX66YmBjFxcVp586datWqlXr16qXjx4//5OdSU1P16KOPqmPHjj87LAAA8DylLiOzZs3SqFGjNHz4cDVr1kzz5s1T5cqVtWjRokt+pqCgQEOGDNHUqVPVoEGD/ykwAADwLKUqI/n5+dqxY4eioqL+9Rt4eysqKkqJiYmX/Ny0adMUGBioESNGXNb3ycvLU3Z2dpEXAADwTKUqI5mZmSooKFBQUFCR8aCgIGVkZJT4mc2bN+u1117TggULLvv7zJgxQ/7+/u5XcHBwaWICAIBryBW9mubs2bMaOnSoFixYoBo1alz25yZNmqSsrCz36/Dhw1cwJQAAsFS+NAfXqFFD5cqV07Fjx4qMHzt2TDVr1ix2/LfffqvU1FT179/fPeZyuX74xuXLa9++fQoNDS32OV9fX/n6+pYmGgAAuEaVamWkQoUKCg8PV3x8vHvM5XIpPj5ekZGRxY5v0qSJdu/eraSkJPdrwIAB6tq1q5KSkjj9AgAASrcyIkkxMTGKjo5WRESEbr31Vs2ePVs5OTkaPny4JGnYsGGqU6eOZsyYoYoVK6pFixZFPh8QECBJxcYBAIAzlbqMDB48WCdOnFBsbKwyMjJ08803a82aNe5NrWlpafL25sauAADg8pS6jEjSmDFjNGbMmBLfS0hI+MnP/uUvf/k53xIAAHgoljAAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADD1s8rISy+9pJCQEFWsWFFt27bVtm3bLnnsggUL1LFjR1WrVk3VqlVTVFTUTx4PAACcpdRlZPny5YqJiVFcXJx27typVq1aqVevXjp+/HiJxyckJOiee+7Rp59+qsTERAUHB6tnz55KT0//n8MDAIBrX6nLyKxZszRq1CgNHz5czZo107x581S5cmUtWrSoxOPffPNN/f73v9fNN9+sJk2aaOHChXK5XIqPj/+fwwMAgGtfqcpIfn6+duzYoaioqH/9Bt7eioqKUmJi4mX9Hrm5ubpw4YKqV69+yWPy8vKUnZ1d5AUAADxTqcpIZmamCgoKFBQUVGQ8KChIGRkZl/V7/OEPf1Dt2rWLFJr/NGPGDPn7+7tfwcHBpYkJAACuIWV6Nc2zzz6rZcuW6f3331fFihUvedykSZOUlZXlfh0+fLgMUwIAgLJUvjQH16hRQ+XKldOxY8eKjB87dkw1a9b8yc/+6U9/0rPPPqt169apZcuWP3msr6+vfH19SxMNAABco0q1MlKhQgWFh4cX2Xz642bUyMjIS35u5syZeuqpp7RmzRpFRET8/LQAAMDjlGplRJJiYmIUHR2tiIgI3XrrrZo9e7ZycnI0fPhwSdKwYcNUp04dzZgxQ5L03HPPKTY2VkuXLlVISIh7b4mfn5/8/Px+wT8KAAC4FpW6jAwePFgnTpxQbGysMjIydPPNN2vNmjXuTa1paWny9v7Xgssrr7yi/Px8/frXvy7y+8TFxenJJ5/839IDAIBrXqnLiCSNGTNGY8aMKfG9hISEIl+npqb+nG8BAAAcgmfTAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABgijICAABMUUYAAIApyggAADBFGQEAAKYoIwAAwBRlBAAAmKKMAAAAU5QRAABg6meVkZdeekkhISGqWLGi2rZtq23btv3k8W+//baaNGmiihUrKiwsTB999NHPCgsAADxPqcvI8uXLFRMTo7i4OO3cuVOtWrVSr169dPz48RKP37p1q+655x6NGDFCu3bt0sCBAzVw4EB9+eWX/3N4AABw7St1GZk1a5ZGjRql4cOHq1mzZpo3b54qV66sRYsWlXj8Cy+8oN69e+uxxx5T06ZN9dRTT+mWW27R3Llz/+fwAADg2le+NAfn5+drx44dmjRpknvM29tbUVFRSkxMLPEziYmJiomJKTLWq1cvrVy58pLfJy8vT3l5ee6vs7KyJEnZ2dmliVvmXHm51hE8xtX+//W1gjn5y2FO/nKYl7+Ma2FO/pixsLDwJ48rVRnJzMxUQUGBgoKCiowHBQVp7969JX4mIyOjxOMzMjIu+X1mzJihqVOnFhsPDg4uTVxcw/xnWycAimJO4mpzLc3Js2fPyt/f/5Lvl6qMlJVJkyYVWU1xuVw6deqUrr/+enl5eRkmu7ZlZ2crODhYhw8fVtWqVa3jAJKYl7j6MCd/OYWFhTp79qxq1679k8eVqozUqFFD5cqV07Fjx4qMHzt2TDVr1izxMzVr1izV8ZLk6+srX1/fImMBAQGliYqfULVqVf4Dw1WHeYmrDXPyl/FTKyI/KtUG1goVKig8PFzx8fHuMZfLpfj4eEVGRpb4mcjIyCLHS9Inn3xyyeMBAICzlPo0TUxMjKKjoxUREaFbb71Vs2fPVk5OjoYPHy5JGjZsmOrUqaMZM2ZIksaPH6/OnTvr+eefV79+/bRs2TJ98cUXmj9//i/7JwEAANekUpeRwYMH68SJE4qNjVVGRoZuvvlmrVmzxr1JNS0tTd7e/1pwadeunZYuXarJkyfr8ccfV6NGjbRy5Uq1aNHil/tT4LL4+voqLi6u2CkwwBLzElcb5mTZ8yr8b9fbAAAAXEE8mwYAAJiijAAAAFOUEQAAYIoyAgAATFFGAACAKcqIh3v99deVm8tDqXD1iIuL06FDh6xjAEUwL21RRjzcxIkTVbNmTY0YMUJbt261jgPob3/7m0JDQ9W9e3ctXbq0yBO6ASvMS1uUEQ+Xnp6uxYsXKzMzU126dFGTJk303HPP/eRTk4ErKSkpSdu3b1fz5s01fvx41axZUw8++KC2b99uHQ0Oxry0xU3PHOTYsWP661//qsWLF2vv3r3q3bu3RowYof79+xe5ay5QVi5cuKAPPvhAr7/+utauXasmTZpoxIgRuv/++y/r4VrAlcC8LHv8DeQgQUFB6tChgyIjI+Xt7a3du3crOjpaoaGhSkhIsI4HByosLNSFCxeUn5+vwsJCVatWTXPnzlVwcLCWL19uHQ8Oxbwse5QRBzh27Jj+9Kc/qXnz5urSpYuys7P14YcfKiUlRenp6brrrrsUHR1tHRMOsmPHDo0ZM0a1atXShAkT1Lp1a3399dfasGGD9u/fr2eeeUbjxo2zjgmHYV7a4TSNh+vfv7/Wrl2rxo0ba+TIkRo2bJiqV69e5Jjjx4+rZs2acrlcRinhJGFhYdq7d6969uypUaNGqX///ipXrlyRYzIzMxUYGMicRJlhXtoq9VN7cW0JDAzUhg0bFBkZecljbrjhBqWkpJRhKjjZXXfdpQceeEB16tS55DE1atTgBz7KFPPSFisjAADAFHtGPNy4ceM0Z86cYuNz587Vww8/XPaB4HiDBg3Sc889V2x85syZ+s1vfmOQCGBeWqOMeLh3331X7du3Lzberl07vfPOOwaJ4HQbN25U3759i4336dNHGzduNEgEMC+tUUY83MmTJ0u8Lr5q1arKzMw0SASnO3funCpUqFBs3MfHR9nZ2QaJAOalNcqIh2vYsKHWrFlTbPzvf/+7GjRoYJAIThcWFlbivRqWLVumZs2aGSQCmJfWuJrGw8XExGjMmDE6ceKEunXrJkmKj4/X888/r9mzZ9uGgyNNmTJFd955p7799tsic/Ktt97S22+/bZwOTsW8tMXVNA7wyiuv6JlnntGRI0ckSSEhIXryySc1bNgw42RwqtWrV2v69OlKSkpSpUqV1LJlS8XFxalz587W0eBgzEs7lBEHOXHihCpVqiQ/Pz/rKAAAuFFGAACAKfaMOMA777yjFStWKC0tTfn5+UXe27lzp1EqOFVBQYH+/Oc/X3JOnjp1yigZnIx5aYuraTzcnDlzNHz4cAUFBWnXrl269dZbdf311+vgwYPq06ePdTw40NSpUzVr1iwNHjxYWVlZiomJ0Z133ilvb289+eST1vHgUMxLW5ym8XBNmjRRXFyc7rnnHlWpUkXJyclq0KCBYmNjderUKc2dO9c6IhwmNDRUc+bMUb9+/VSlShUlJSW5xz777DMtXbrUOiIciHlpi5URD5eWlqZ27dpJkipVqqSzZ89KkoYOHaq33nrLMhocKiMjQ2FhYZIkPz8/ZWVlSZJuu+02rV692jIaHIx5aYsy4uFq1qzpPtd544036rPPPpMkpaSkiEUxWKhbt66OHj0q6Yd/jX788ceSpO3bt8vX19cyGhyMeWmLMuLhunXrplWrVkmShg8frgkTJqhHjx4aPHiw7rjjDuN0cKI77rhD8fHxkqSxY8dqypQpatSokYYNG6YHHnjAOB2cinlpiz0jHs7lcsnlcql8+R8unFq2bJm2bt2qRo0aafTo0SU+iwEoS5999pl7Tvbv3986DiCJeVnWKCMe7OLFi5o+fboeeOAB1a1b1zoOoAsXLmj06NGaMmWK6tevbx0HkMS8vBpQRjycn5+fvvzyS4WEhFhHASRJ/v7+SkpK4oc+rirMS1vsGfFw3bt314YNG6xjAG4DBw7UypUrrWMARTAvbXEHVg/Xp08fTZw4Ubt371Z4eLiuu+66Iu8PGDDAKBmcqlGjRpo2bZq2bNlS4pwcN26cUTI4GfPSFqdpPJy396UXv7y8vFRQUFCGaQD95DK4l5eXDh48WIZpgB8wL21RRgAAgCn2jAAAAFPsGfFw06ZN+8n3Y2NjyygJ8IP/dgOpRYsWlVES4F+Yl7YoIx7u/fffL/L1hQsXlJKSovLlyys0NJQygjJ3+vTpIl9fuHBBX375pc6cOaNu3boZpYLTMS9tUUY83K5du4qNZWdn6/777+d28DDxnwVZ+uFOwQ8++KBCQ0MNEgHMS2tsYHWo3bt3q3///kpNTbWOAkiS9u3bpy5durgfVgZcDZiXZYMNrA6VlZXlfkQ2cDX49ttvdfHiResYQBHMy7LBaRoPN2fOnCJfFxYW6ujRo3rjjTfUp08fo1RwspiYmCJf/zgnV69erejoaKNUcDrmpS1O03i4/7yRj7e3t2644QZ169ZNkyZNUpUqVYySwam6du1a5Ot/n5MPPPCA+wnTQFliXtqijAAAAFPsGfFwWVlZOnXqVLHxU6dOKTs72yARnC4lJUX79+8vNr5//342VMMM89IWZcTD3X333Vq2bFmx8RUrVujuu+82SASnu//++7V169Zi459//rnuv//+sg8EiHlpjdM0Hq569erasmWLmjZtWmR87969at++vU6ePGmUDE5VtWpV7dy5Uw0bNiwyfuDAAUVEROjMmTM2weBozEtbrIx4uLy8vBIvS7tw4YLOnz9vkAhO5+XlpbNnzxYbz8rK4inSMMO8tEUZ8XC33nqr5s+fX2x83rx5Cg8PN0gEp+vUqZNmzJhR5Ad8QUGBZsyYoQ4dOhgmg5MxL21xmsbDbdmyRVFRUWrTpo26d+8uSYqPj9f27dv18ccfq2PHjsYJ4TR79uxRp06dFBAQ4J5/mzZtUnZ2ttavX68WLVoYJ4QTMS9tUUYcICkpSX/84x+VlJSkSpUqqWXLlpo0aZIaNWpkHQ0OdeTIEc2dO1fJycnuOTlmzBhVr17dOhocjHlphzICAABMsWfEw3300Udau3ZtsfG1a9fq73//u0EiON3rr7+ut99+u9j422+/rcWLFxskApiX1igjHm7ixIkl7gQvLCzUxIkTDRLB6WbMmKEaNWoUGw8MDNT06dMNEgHMS2uUEQ+3f/9+NWvWrNh4kyZNdODAAYNEcLq0tLRiz0ySpHr16iktLc0gEcC8tEYZ8XD+/v46ePBgsfEDBw7ouuuuM0gEpwsMDNQ//vGPYuPJycm6/vrrDRIBzEtrlBEPd/vtt+vhhx/Wt99+6x47cOCAHnnkEQ0YMMAwGZzqnnvu0bhx4/Tpp5+qoKBABQUFWr9+vcaPH88jCmCGeWmLq2k8XFZWlnr37q0vvvhCdevWlSR999136tixo9577z0FBATYBoTj5Ofna+jQoXr77bfdj2V3uVwaNmyY5s2bpwoVKhgnhBMxL21RRhygsLBQn3zySZFr5zt16mQdCw73zTffuOdkWFiY6tWrZx0JYF4aoYwAAABT5a0D4MrLycnRhg0blJaWpvz8/CLvjRs3zigVnOy7777TqlWrSpyTs2bNMkoFp2Ne2qGMeLhdu3apb9++ys3NVU5OjqpXr67MzExVrlxZgYGBlBGUufj4eA0YMEANGjTQ3r171aJFC6WmpqqwsFC33HKLdTw4FPPSFlfTeLgJEyaof//+On36tCpVqqTPPvtMhw4dUnh4uP70pz9Zx4MDTZo0SY8++qh2796tihUr6t1339Xhw4fVuXNn/eY3v7GOB4diXtpiz4iHCwgI0Oeff66bbrpJAQEBSkxMVNOmTfX5558rOjpae/futY4Ih6lSpYqSkpIUGhqqatWqafPmzWrevLmSk5N1++23KzU11ToiHIh5aYuVEQ/n4+Mjb+8f/m8ODAx030nQ399fhw8ftowGh7ruuuvc5+Nr1apV5B44mZmZVrHgcMxLW+wZ8XCtW7fW9u3b1ahRI3Xu3FmxsbHKzMzUG2+8oRYtWljHgwP96le/0ubNm9W0aVP17dtXjzzyiHbv3q333ntPv/rVr6zjwaGYl7Y4TePhvvjiC509e1Zdu3bV8ePHNWzYMG3dulWNGjXSokWL1KpVK+uIcJiDBw/q3LlzatmypXJycvTII4+45+SsWbO4rwNMMC9tUUYgSdqyZYsiIiLk6+trHQWQJL311lsaMGAAz1DCVYV5eWVQRiBJqlq1qpKSktSgQQPrKIAk5iSuTszLK4MNrJD0wy3jgasJcxJXI+bllUEZAQAApigjAADAFGUEAACYooxAkuTl5WUdAQDgUJQRSGJTFq4+9erVk4+Pj3UMoAjm5ZVBGXGAixcvat26dXr11Vd19uxZSdKRI0d07tw59zFnz57lUjWUiQYNGujkyZPFxs+cOVNkDn755ZcKDg4uy2hwMOalLW4H7+EOHTqk3r17Ky0tTXl5eerRo4eqVKmi5557Tnl5eZo3b551RDhMamqqCgoKio3n5eUpPT3dIBHAvLRGGfFw48ePV0REhJKTk3X99de7x++44w6NGjXKMBmcZtWqVe5fr127Vv7+/u6vCwoKFB8fr5CQEINkcDLm5dWBO7B6uOuvv15bt27VTTfdpCpVqig5OVkNGjRQamqqmjVrptzcXOuIcIgfnx7t5eVVbI+Sj4+PQkJC9Pzzz+u2226ziAeHYl5eHVgZ8XAul6vEpcfvvvtOVapUMUgEp3K5XJKk+vXra/v27apRo4ZxIoB5ebVgA6uH69mzp2bPnu3+2svLS+fOnVNcXJz69u1rFwyOdfDgwUv+wGelDlaYl7YoIx7u+eef15YtW9SsWTN9//33uvfeexUSEqL09HQ999xz1vHgQFFRUSVuCPz888918803l30gQMxLa5QRD1e3bl0lJyfr8ccf14QJE9S6dWs9++yz2rVrlwIDA63jwYEqVqyoli1bavny5ZJ+WCZ/8skn1bFjR1brYIZ5aYsNrADK3EsvvaT/+7//0+23367U1FQdOnRIr7/+unr27GkdDQ7GvLRDGfFw/37Z2r/z8vJSxYoV1bBhQ9WvX7+MUwHSpEmT9Nxzz6l8+fJKSEhQu3btrCMBzEsjlBEP5+3tXeIlaz+OeXl5qUOHDlq5cqWqVatmlBJOcvr0aY0cOVLx8fH64x//qA0bNmjlypWaOXOmfv/731vHg0MxL22xZ8TDffLJJ2rTpo0++eQTZWVlKSsrS5988onatm2rDz/8UBs3btTJkyf16KOPWkeFQ7Ro0ULHjh3Trl27NGrUKP31r3/Va6+9pilTpqhfv37W8eBQzEtjhfBozZs3L9yyZUux8c2bNxc2a9assLCwsPCTTz4pDA4OLutocKhp06YVFhQUFBs/fPhwYVRUlEEigHlpjdM0Hq5SpUravn27WrRoUWR89+7duvXWW3X+/HkdOnRITZs25Vp6lLnvv/9eFStWtI4BFMG8LHucpvFw4eHheuyxx3TixAn32IkTJ/R///d/atOmjSRp//79PIUSZcblcumpp55SnTp15Ofnp4MHD0qSpkyZotdee804HZyKeWmLMuLhXnvtNaWkpKhu3bpq2LChGjZsqLp16yo1NVULFy6UJJ07d06TJ082TgqnePrpp/WXv/xFM2fOVIUKFdzjLVq0cM9JoKwxL21xmsYBXC6XPv74Y33zzTeSpJtuukk9evRwPyAKKEsNGzbUq6++qu7duxd5eOPevXsVGRmp06dPW0eEAzEvbfGgPAfw9vZW79691bt3b+sogNLT09WwYcNi4y6XSxcuXDBIBDAvrVFGHCAnJ0cbNmxQWlqa8vPzi7w3btw4o1RwqmbNmmnTpk2qV69ekfF33nlHrVu3NkoFp2Ne2qKMeLhdu3apb9++ys3NVU5OjqpXr67MzExVrlxZgYGBlBGUudjYWEVHRys9PV0ul0vvvfee9u3bpyVLlujDDz+0jgeHYl7aYs+Ih+vSpYsaN26sefPmyd/fX8nJyfLx8dF9992n8ePH684777SOCAfatGmTpk2bpuTkZJ07d0633HKLYmNjeQYITDEv7VBGPFxAQIA+//xz3XTTTQoICFBiYqKaNm2qzz//XNHR0dq7d691RACAw3E5hYfz8fFxXzUTGBiotLQ0SZK/v78OHz5sGQ0O1aBBA508ebLY+JkzZ9SgQQODRADz0hp7Rjxc69attX37djVq1EidO3dWbGysMjMz9cYbbxS7KytQFlJTU1VQUFBsPC8vT+np6QaJAOalNcqIh5s+fbrOnj0rSXrmmWc0bNgwPfjgg2rUqJEWLVpknA5OsmrVKvev165dK39/f/fXBQUFio+PV0hIiEEyOBnz8urAnhEAZeLH04VeXl76zx87Pj4+CgkJ0fPPP6/bbrvNIh4cinl5daCMAChT9evX1/bt21WjRg3rKIAb89IWG1g93MmTJ/XQQw+pWbNmqlGjhqpXr17kBZS1lJSUy/qBHxYWxiZrlBnmpS32jHi4oUOH6sCBAxoxYoSCgoLk5eVlHQm4LKmpqdyGG1cd5uWVQRnxcJs2bdLmzZvVqlUr6ygAAJSI0zQerkmTJjp//rx1DAAALoky4uFefvllPfHEE9qwYYNOnjyp7OzsIi8AAKxxmsbDBQQEKDs7W926dSsyXlhYKC8vrxJv8gMAQFmijHi4IUOGyMfHR0uXLmUDKwDgqkQZ8XBffvmldu3apZtuusk6CiBJWrJkiQYPHixfX98i4/n5+Vq2bJmGDRsmSXr11VcVFBRkERG4JObllcFNzzxcp06dFBsbq6ioKOsogCSpXLlyOnr0qAIDA4uMnzx5UoGBgZw6RJmZM2fOZR87bty4K5gElBEP9/bbb+vJJ5/UY489prCwMPn4+BR5v2XLlkbJ4FTe3t46duyYbrjhhiLjycnJ6tq1q06dOmWUDE5Tv379Il+fOHFCubm5CggIkPTDE3srV66swMBAHTx40CChc1BGPNyPz134dz8+g4ENrChLrVu3lpeXl5KTk9W8eXOVL/+vs8QFBQVKSUlR7969tWLFCsOUcKqlS5fq5Zdf1muvveY+rb1v3z6NGjVKo0eP1pAhQ4wTejbKiIc7dOjQT75fr169MkoCp5s6dar7fx955BH5+fm536tQoYJCQkI0aNAgVahQwSoiHCw0NFTvvPOOWrduXWR8x44d+vWvf62UlBSjZM7ABlYPR9nA1SIuLk6SFBISosGDB6tixYrGiYB/OXr0qC5evFhsvKCgQMeOHTNI5CysjHigVatWqU+fPvLx8dGqVat+8tgBAwaUUSqgqB07dujrr7+WJDVv3rzYv0iBstS/f3+lp6dr4cKFuuWWWyT9MEd/+9vfqk6dOv/1Zyn+N5QRD+Tt7a2MjAwFBgaWuGfkR+wZgYXjx4/r7rvvVkJCQpGNgl27dtWyZcuKbWwFysKJEycUHR2tNWvWuDf6X7x4Ub169dJf/vKXYld/4ZdFGQFQpgYPHqyDBw9qyZIlatq0qSRpz549io6OVsOGDfXWW28ZJ4STffPNN9q7d6+kH57t1bhxY+NEzkAZgSQpLCxMH330kYKDg62jwMP5+/tr3bp1atOmTZHxbdu2qWfPnjpz5oxNMEA/3HwvJSVFoaGhRa74wpXFg/IgSUpNTdWFCxesY8ABXC5XsfvdSJKPj49cLpdBIkDKzc3ViBEjVLlyZTVv3lxpaWmSpLFjx+rZZ581Tuf5KCMAylS3bt00fvx4HTlyxD2Wnp6uCRMmqHv37obJ4GSTJk1ScnKyEhISilzpFRUVpeXLlxsmcwbKCIAyNXfuXGVnZyskJEShoaEKDQ1V/fr1lZ2drRdffNE6Hhxq5cqVmjt3rjp06FDkgaLNmzfXt99+a5jMGTghBqBMBQcHa+fOnVq3bp17o2DTpk15fhJMnThxosQrZnJycnjaeRmgjAAoc15eXurRo4d69OhhHQWQJEVERGj16tUaO3asJLkLyMKFCxUZGWkZzREoIwDKXHx8vOLj43X8+PFim1YXLVpklApONn36dPXp00d79uzRxYsX9cILL2jPnj3aunWrNmzYYB3P47FnxMMtWbJEeXl5xcbz8/O1ZMkS99evvvqqgoKCyjIaHGrq1Knq2bOn4uPjlZmZqdOnTxd5ARY6dOigpKQkXbx4UWFhYfr4448VGBioxMREhYeHW8fzeNxnxMOVK1dOR48eLXYu9OTJkwoMDOQOrChztWrV0syZMzV06FDrKACuEpym8XCFhYUlbr767rvv5O/vb5AITpefn6927dpZxwCUnZ192cdWrVr1CiYBZcRDtW7dWl5eXvLy8lL37t2L3EmwoKBAKSkp6t27t2FCONXIkSO1dOlSTZkyxToKHC4gIOC/Xinz4z/oWEW+sigjHmrgwIGSpKSkJPXq1Ut+fn7u9ypUqKCQkBANGjTIKB2cJiYmxv1rl8ul+fPna926dWrZsmWxu7HOmjWrrOPBoT799FPrCPgn9ox4uMWLF2vw4MFF7igIlLWuXbte1nFeXl5av379FU4D4GpDGXGIHTt26Ouvv5b0wx0FW7dubZwIAGz94x//uOxjW7ZseQWTgDLi4Y4fP667775bCQkJCggIkCSdOXNGXbt21bJly3TDDTfYBoTjZWdna/369WrSpImaNGliHQcO4u3tLS8vL/23vwbZM3LlsWfEw40dO1Znz57VV199paZNm0qS9uzZo+joaI0bN05vvfWWcUI4zV133aVOnTppzJgxOn/+vCIiIpSamqrCwkItW7aMvUwoMykpKdYR8E+sjHg4f39/rVu3Tm3atCkyvm3bNvXs2VNnzpyxCQbHqlmzptauXatWrVpp6dKliouLU3JyshYvXqz58+dr165d1hEBlDHuwOrhXC5XsasVJMnHx6fYbbiBspCVlaXq1atLktasWaNBgwapcuXK6tevn/bv32+cDk72xhtvqH379qpdu7YOHTokSZo9e7b+9re/GSfzfJQRD9etWzeNHz9eR44ccY+lp6drwoQJ6t69u2EyOFVwcLASExOVk5OjNWvWqGfPnpKk06dPc9UXzLzyyiuKiYlR3759debMGfcekYCAAM2ePds2nANQRjzc3LlzlZ2drZCQEIWGhio0NFT169dXdna2XnzxRet4cKCHH35YQ4YMUd26dVW7dm116dJFkrRx40aFhYXZhoNjvfjii1qwYIGeeOIJlStXzj0eERGh3bt3GyZzBvaMOEBhYaHWrVunvXv3SpKaNm2qqKgo41Rwsh07digtLU09evRw35Bv9erVCggIUPv27Y3TwYkqVaqkvXv3ql69eqpSpYqSk5PVoEED7d+/Xy1bttT58+etI3o0rqZxAC8vL/Xo0UM9evSwjgJIksLDw4s9CbVfv35Fvq5ataqSkpLUoEGDsowGh6pfv76SkpJUr169IuNr1qxxX4mIK4cy4gDx8fGKj4/X8ePHi21aXbRokVEq4KexaIuyFBMTo4ceekjff/+9CgsLtW3bNr311luaMWOGFi5caB3P41FGPNzUqVM1bdo0RUREqFatWv/1oVAA4EQjR45UpUqVNHnyZOXm5uree+9V7dq19cILL+juu++2jufx2DPi4WrVqqWZM2dq6NCh1lGAUvn38/ZAWcrNzdW5c+cUGBhoHcUxWBnxcPn5+WrXrp11DAC4Jhw/flz79u2T9MN+Ox6ZUTa4tNfDjRw5UkuXLrWOAZQapxRRls6ePauhQ4eqdu3a6ty5szp37qzatWvrvvvuU1ZWlnU8j8fKiAeKiYlx/9rlcmn+/Plat26dWrZsWexurLNmzSrreMBl4QwyytLIkSO1a9curV69WpGRkZKkxMREjR8/XqNHj9ayZcuME3o29ox4oK5du17WcV5eXlq/fv0VTgNc2o8/fkpaBdm8ebPatGkjX1/fso4FB7ruuuu0du1adejQocj4pk2b1Lt3b+Xk5BglcwZWRjzQp59+ah0B+ElLlizRH//4R/ezaBo3bqzHHnusyEbr//xLAbiSrr/+evn7+xcb9/f3V7Vq1QwSOQt7RhwmOztbK1eudN+NFShrs2bN0oMPPqi+fftqxYoVWrFihXr37q3f/e53+vOf/2wdDw41efJkxcTEKCMjwz2WkZGhxx57TFOmTDFM5gycpvFwd911lzp16qQxY8bo/PnzatWqlVJTU1VYWKhly5Zp0KBB1hHhMPXr19fUqVM1bNiwIuOLFy/Wk08+qZSUFKNkcJrWrVsXOUW4f/9+5eXl6cYbb5QkpaWlydfXV40aNdLOnTutYjoCp2k83MaNG/XEE09Ikt5//30VFhbqzJkzWrx4sZ5++mnKCMrc0aNHS7zcvF27djp69KhBIjjVwIEDrSPgnygjHi4rK0vVq1eX9MMzFgYNGqTKlSurX79+euyxx4zTwYkaNmyoFStW6PHHHy8yvnz5cjVq1MgoFZwoLi7OOgL+iTLi4YKDg5WYmKjq1atrzZo17svTTp8+rYoVKxqngxNNnTpVgwcP1saNG91P6N2yZYvi4+O1YsUK43QALFBGPNzDDz+sIUOGyM/PT/Xq1VOXLl0k/XD6JiwszDYcHGnQoEHatm2bZs2apZUrV0qSmjZtqm3btql169a24eBYBQUF+vOf/6wVK1YoLS1N+fn5Rd4/deqUUTJnYAOrA+zYsUNpaWnq0aOH/Pz8JEmrV69WQECA+1+mQFm4cOGCRo8erSlTpqh+/frWcQC32NhYLVy4UI888ogmT56sJ554QqmpqVq5cqViY2M1btw464gejTICSVLVqlWVlJTEQ8lwxfn7+yspKYkygqtKaGio5syZo379+qlKlSpKSkpyj3322Wc8VuMK4z4jkMStt1F2Bg4c6D49A1wtMjIy3Keu/fz83M+jue2227R69WrLaI7AnhEAZapRo0aaNm2atmzZovDwcF133XVF3mc5HBbq1q2ro0eP6sYbb1RoaKg+/vhj3XLLLdq+fTuPJCgDnKaBJKlKlSpKTk7mNA2uuJ86PePl5aWDBw+WYRrgBxMnTlTVqlX1+OOPa/ny5brvvvsUEhKitLQ0TZgwQc8++6x1RI9GGYEkyggA/LvExEQlJiaqUaNG6t+/v3Ucj0cZgSQ2sKLs5efnKyUlRaGhoSpfnjPGgJPxEwCS2MCKspObm6uxY8dq8eLFkqRvvvlGDRo00NixY1WnTh1NnDjROCGcYtWqVZd97IABA65gElBGHOTHwvHvD4b60d///nfVqVOnrCPBgSZNmqTk5GQlJCSod+/e7vGoqCg9+eSTlBGUmct9No2Xl5cKCgqubBiH49JeB1iyZInCwsJUqVIlVapUSS1bttQbb7xR5JgOHTqwYxxlYuXKlZo7d646dOhQpBg3b95c3377rWEyOI3L5bqsF0XkymNlxMPNmjVLU6ZM0ZgxY9x3W928ebN+97vfKTMzUxMmTDBOCKc5ceKEAgMDi43n5OSUuGoHlLXvv/+eZ3eVMVZGPNyLL76oV155Rc8995wGDBigAQMGaObMmXr55Zc1Z84c63hwoIiIiCI3kfqxgCxcuFCRkZFWseBwBQUFeuqpp1SnTh35+fm5LzGfMmWKXnvtNeN0no+VEQ939OhRtWvXrth4u3btdPToUYNEcLrp06erT58+2rNnjy5evKgXXnhBe/bs0datW7VhwwbreHCoZ555RosXL9bMmTM1atQo93iLFi00e/ZsjRgxwjCd52NlxMM1bNiwxMeyL1++XI0aNTJIBKfr0KGDkpOTdfHiRYWFhenjjz9WYGCgEhMTFR4ebh0PDrVkyRLNnz9fQ4YMUbly5dzjrVq10t69ew2TOQMrIx5u6tSpGjx4sDZu3OjeM7JlyxbFx8eXWFKAK61bt27q3LmzFixYUGT89OnT6tatm9avX2+UDE6Wnp6uhg0bFht3uVy6cOGCQSJnYWXEww0aNEjbtm1TjRo1tHLlSq1cuVI1atTQtm3bdMcdd1jHgwMlJCRo7ty5GjhwoHJzc93j+fn5nKaBmWbNmmnTpk3Fxt955x21bt3aIJGzsDLiwS5cuKDRo0drypQp+utf/2odB3Bbt26dRo8erbZt2+qDDz5QSEiIdSQ4XGxsrKKjo5Weni6Xy6X33ntP+/bt05IlS/Thhx9ax/N4rIx4MB8fH7377rvWMYBiatWqpQ0bNigsLExt2rRRQkKCdSQ43O23364PPvhA69at03XXXafY2Fh9/fXX+uCDD9SjRw/reB6PMuLhBg4cqJUrV1rHANx+vJTX19dXS5cu1fjx49W7d2+9/PLLxsngVBcvXtS0adNUv359ffLJJzp+/Lhyc3O1efNm9ezZ0zqeI/CgPA/39NNP6/nnn1f37t0VHh6u6667rsj748aNM0oGp/L29lZGRkaRG5+9++67io6O1vnz57nbJUz4+fnpyy+/5JShEcqIh6tfv/4l3/Py8nLf2AcoK4cOHdKNN95Y7G6rX331lb744gtFR0cbJYOT3X777brzzjuZf0YoIwAAx5s3b56mTp2qIUOGlLiKzFN7ryzKiEPk5+crJSVFoaGhKl+ei6gA4N95e196CyVP7b3y2MDq4XJzczVixAhVrlxZzZs3V1pamiRp7NixevbZZ43TAcDVgaf22qKMeLhJkyYpOTlZCQkJRZ5CGRUVpeXLlxsmA4BrT1hYmA4fPmwdw+OwXu/hVq5cqeXLl+tXv/pVkQ2DzZs317fffmuYDACuPampqdwe/gpgZcTDnThxosgllD/KyckpdjUDAAAWKCMeLiIiQqtXr3Z//WMBWbhwoSIjI61iAQDgxmkaDzd9+nT16dNHe/bs0cWLF/XCCy9oz5492rp1Kw8lAwBcFVgZ8XAdOnRQcnKyLl68qLCwMH388ccKDAxUYmKiwsPDreMBAMDKiKfr1q2bOnfurAULFhQZP336tLp166b169cbJQMA4AesjHi4hIQEzZ07VwMHDlRubq57PD8/n9M0APBPS5YsUV5eXrHx/Px8LVmyxP31q6++qqCgoLKM5gjcgdXDeXt7a9euXRo9erRycnL0wQcfKCQkRMeOHVPt2rW5mQ8ASCpXrpyOHj1a7OrDkydPKjAwkJ+VVxgrIw5Qq1YtbdiwQWFhYWrTpo0SEhKsIwHAVaWwsLDE2x1899138vf3N0jkLOwZ8XA//sfl6+urpUuX6umnn1bv3r31hz/8wTgZANhr3bq1vLy85OXlpe7duxd5dldBQYFSUlLUu3dvw4TOQBnxcP95Fm7y5Mlq2rQpj8kGAEkDBw6UJCUlJalXr17y8/Nzv1ehQgWFhIRo0KBBRumcgz0jHu7QoUO68cYbiy0/fvXVV/riiy8oJQAgafHixRo8eHCRZ3ih7FBGAAD4px07dujrr7+W9MMzvFq3bm2cyBk4TQMAcLzjx4/r7rvvVkJCggICAiRJZ86cUdeuXbVs2TLdcMMNtgE9HFfTAAAcb+zYsTp79qy++uornTp1SqdOndKXX36p7OxsjRs3zjqex+M0DQDA8fz9/bVu3Tq1adOmyPi2bdvUs2dPnTlzxiaYQ7AyAgBwPJfLJR8fn2LjPj4+crlcBomchTICAHC8bt26afz48Tpy5Ih7LD09XRMmTFD37t0NkzkDp2kAAI53+PBhDRgwQF999ZWCg4PdYy1atNCqVatUt25d44SejTICAIB+uEnkunXrtHfvXklS06ZNFRUVZZzKGSgjAADAFPcZAQBAUnx8vOLj43X8+PFim1YXLVpklMoZKCMAAMebOnWqpk2bpoiICNWqVavEJ/jiyuE0DQDA8WrVqqWZM2dq6NCh1lEciUt7AQCOl5+fr3bt2lnHcCzKCADA8UaOHKmlS5dax3AsTtMAABwpJibG/WuXy6XFixerZcuWatmyZbG7sc6aNaus4zkKZQQA4Ehdu3a9rOO8vLy0fv36K5zG2SgjAADAFHtGAAD4D9nZ2Vq5cqX7bqy4sigjAADHu+uuuzR37lxJ0vnz5xUREaG77rpLYWFhevfdd43TeT7KCADA8TZu3KiOHTtKkt5//30VFhbqzJkzmjNnjp5++mnjdJ6PMgIAcLysrCxVr15dkrRmzRoNGjRIlStXVr9+/bR//37jdJ6PMgIAcLzg4GAlJiYqJydHa9asUc+ePSVJp0+fVsWKFY3TeT6eTQMAcLyHH35YQ4YMkZ+fn+rVq6cuXbpI+uH0TVhYmG04B+DSXgAAJH3xxRc6fPiwevToIT8/P0nS6tWrFRAQoPbt2xun82yUEQCA4x08eFANGjSwjuFYlBEAgON5e3urbt266ty5s7p06aLOnTurYcOG1rEcgzICAHC89PR0JSQkaMOGDdqwYYP279+v2rVrq3PnzuratatGjhxpHdGjUUYAAPgP+/fv1zPPPKM333xTLpdLBQUF1pE8GlfTAAAcLzc3V5s3b1ZCQoISEhK0a9cuNWnSRGPGjHFfWYMrh5URAIDjVahQQdWqVdOQIUPUpUsXdezYUdWqVbOO5RisjAAAHK9v377avHmzli1bpoyMDGVkZKhLly5q3LixdTRHYGUEAIB/+sc//uHexLpp0yaVL19eXbp00ZtvvmkdzaNRRgAA+KfCwkLt2rVLn376qT799FOtXbtWhYWFunjxonU0j8azaQAAjjdr1iwNGDBA119/vdq2bau33npLjRs31rvvvqsTJ05Yx/N4rIwAAByvTZs27huedezYUf7+/taRHIUyAgAATHGaBgAASZs2bdJ9992nyMhIpaenS5LeeOMNbd682TiZ56OMAAAc791331WvXr1UqVIl7dq1S3l5eZKkrKwsTZ8+3Tid56OMAAAc7+mnn9a8efO0YMEC+fj4uMfbt2+vnTt3GiZzBsoIAMDx9u3bp06dOhUb9/f315kzZ8o+kMNQRgAAjlezZk0dOHCg2PjmzZvVoEEDg0TOQhkBADjeqFGjNH78eH3++efy8vLSkSNH9Oabb+rRRx/Vgw8+aB3P4/FsGgCAI/3jH/9QixYt5O3trUmTJsnlcql79+7Kzc1Vp06d5Ovrq0cffVRjx461jurxuM8IAMCRypUrp6NHjyowMFANGjTQ9u3bVaVKFR04cEDnzp1Ts2bN5OfnZx3TEVgZAQA4UkBAgFJSUhQYGKjU1FS5XC5VqFBBzZo1s47mOJQRAIAjDRo0SJ07d1atWrXk5eWliIgIlStXrsRjDx48WMbpnIUyAgBwpPnz5+vOO+/UgQMHNG7cOI0aNUpVqlSxjuVI7BkBADje8OHDNWfOHMqIEcoIAAAwxX1GAACAKcoIAAAwRRkBAACmKCMAAMAUZQQAAJiijAAAAFOUEQAAYIoyAgAATP0/52uwjS/y/4sAAAAASUVORK5CYII=",
+ "text/plain": [
+ "
\n",
+ "\n",
+ "Tip\n",
+ " \n",
+ "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
+ "\n",
+ "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
+ "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter Notebook tool of your choice.\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "S_JeXXsL1w4a"
+ },
+ "source": [
+ "## Setup\n",
+ "\n",
+ "To complete this tutorial, you will need to install the Argilla client and a few third-party libraries using `pip`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "id": "SbyuZvarlToD"
+ },
+ "outputs": [],
+ "source": [
+ "%pip install transformers argilla datasets torch setfit -qqqqqqq"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "XMxT5_tK346_"
+ },
+ "source": [
+ "The imports needed:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "id": "W4_oh-TmnOVm"
+ },
+ "outputs": [],
+ "source": [
+ "import argilla as rg\n",
+ "from datasets import load_dataset\n",
+ "from transformers import pipeline\n",
+ "from argilla.metrics.text_classification import f1\n",
+ "import pandas as pd"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "shy57P-F48iH"
+ },
+ "source": [
+ "If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "id": "klxDkoErnaPI"
+ },
+ "outputs": [],
+ "source": [
+ "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
+ "# Replace api_key if you configured a custom API key\n",
+ "# Replace workspace with the name of your workspace\n",
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"owner.apikey\", workspace=\"admin\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Set the HF_TOKEN environment variable\n",
+ "# import os\n",
+ "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
+ "\n",
+ "# # Replace api_url with the url to your HF Spaces URL\n",
+ "# # Replace api_key if you configured a custom API key\n",
+ "# # Replace workspace with the name of your workspace\n",
+ "# rg.init(\n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
+ "# api_key=\"owner.apikey\",\n",
+ "# workspace=\"admin\",\n",
+ "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
+ "# )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "MQibTErO5De0"
+ },
+ "source": [
+ "For this tutorial the HugginFace [ag_news](https://huggingface.co/datasets/ag_news) dataset is chosen:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "4AbgSHtx1Hbh"
- },
- "source": [
- "## Introduction\n",
- "When working on Text Classification, you may want to compare two models to decide which one to use.\n",
- "For this, we compute the F1 score on train models using their annotations as the true text class.\n",
- "The F1 score can be interpreted as a harmonic mean of the precision and recall, where an F1 score reaches its best value at 1 and worst score at 0. (more info in this [documentation](https://scikit-learn.org/stable/modules/generated/sklearn.metrics.f1_score.html))\n",
- "\n",
- "*Argilla* allows you to deploy and monitor any model you like, but in this tutorial, we will focus on the two most common frameworks in the NLP space: [transformers](https://huggingface.co/docs/transformers) and [SetFit](https://github.com/huggingface/setfit). Let's get started!"
- ]
+ "id": "teaiKrR1ng15",
+ "outputId": "381f5715-9f37-46b8-b45f-ec767c906fc7"
+ },
+ "outputs": [],
+ "source": [
+ "news_dataset = load_dataset(\"ag_news\", split=\"test\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "sA2WK_px6pa1"
+ },
+ "source": [
+ "This dataset is composed of two columns, one is the text of the news article and the other one is the label associated with this text article:\n",
+ "\n",
+ "*For this tutorial, we will consider the label as the annotation of the text*\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "blvo7FZz8ccU"
+ },
+ "source": [
+ "We transform our dataset in order to create an argilla TextClassificationDataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "qX1M5cMP1XyX"
- },
- "source": [
- "## Running Argilla\n",
- "\n",
- "For this tutorial, you will need to have an Argilla server running. There are two main options for deploying and running Argilla:\n",
- "\n",
- "\n",
- "**Deploy Argilla on Hugging Face Spaces**: If you want to run tutorials with external notebooks (e.g., Google Colab) and you have an account on Hugging Face, you can deploy Argilla on Spaces with a few clicks:\n",
- "\n",
- "[](https://huggingface.co/new-space?template=argilla/argilla-template-space)\n",
- "\n",
- "For details about configuring your deployment, check the [official Hugging Face Hub guide](https://huggingface.co/docs/hub/spaces-sdks-docker-argilla).\n",
- "\n",
- "\n",
- "**Launch Argilla using Argilla's quickstart Docker image**: This is the recommended option if you want [Argilla running on your local machine](../../getting_started/quickstart.ipynb). Note that this option will only let you run the tutorial locally and not with an external notebook service.\n",
- "\n",
- "For more information on deployment options, please check the Deployment section of the documentation.\n",
- "\n",
- "
\n",
- "\n",
- "Tip\n",
- " \n",
- "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
- "\n",
- "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
- "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter Notebook tool of your choice.\n",
- "
"
- ]
+ "id": "Ttt0aUJ7pYzz",
+ "outputId": "15f7798d-6ad7-45f5-82a6-225088ee0076"
+ },
+ "outputs": [],
+ "source": [
+ "int_to_label = {\n",
+ " 0: \"World\",\n",
+ " 1: \"Sports\",\n",
+ " 2: \"Business\",\n",
+ " 3: \"Sci/Tech\",\n",
+ "}\n",
+ "\n",
+ "news_dataset = news_dataset.map(\n",
+ " lambda row: {\"prediction\": [{\"label\": int_to_label[row[\"label\"]], \"score\": 1}]}\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
- {
- "cell_type": "markdown",
- "metadata": {
- "id": "S_JeXXsL1w4a"
- },
- "source": [
- "## Setup\n",
- "\n",
- "To complete this tutorial, you will need to install the Argilla client and a few third-party libraries using `pip`:"
- ]
+ "id": "u687F37SoFn8",
+ "outputId": "3f111e4d-97c4-41fe-b595-80b4ccda4520"
+ },
+ "outputs": [],
+ "source": [
+ "ds_record = rg.read_datasets(dataset=news_dataset, task=\"TextClassification\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Enable Telemetry\n",
+ "\n",
+ "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
+ " tutorial_running()\n",
+ "except ImportError:\n",
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "TEyjYZHAIV9h"
+ },
+ "source": [
+ "## Make Zero Shot Text Classification predictions using transformers\n",
+ "\n",
+ "On HugginFace we choose the model `cross-encoder/nli-distilroberta-base` that is trained to perform zero-shot classification.\n",
+ "We create a pipeline with this model and then perform prediction.\n",
+ "\n",
+ "*note: `device=0` in pipeline() permits to use GPU if you do not have a GPU available delete this parameter*\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "id": "UW4am9A7n4mI"
+ },
+ "outputs": [],
+ "source": [
+ "labels = [\"Sports\", \"Sci/Tech\", \"Business\", \"World\"]\n",
+ "\n",
+ "pipe = pipeline(\n",
+ " \"zero-shot-classification\", model=\"cross-encoder/nli-distilroberta-base\", device=0\n",
+ ")\n",
+ "result = []\n",
+ "with pipe.device_placement():\n",
+ " result = pipe(\n",
+ " [data.text for data in ds_record],\n",
+ " candidate_labels=labels,\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "BnT8GZmWKl7S"
+ },
+ "source": [
+ "Now that predictions are successfully made with the zero-shot model we can transform it into a list of argilla TextClassificationRecord and upload it to our argilla client"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 85,
+ "referenced_widgets": [
+ "03352da397fc457fa70dd5e489cc4844",
+ "9424d0b9953b43638b5f0623ff225d79",
+ "4e8ec2dc3b564cae937741ac7d47372a",
+ "58a2528e6a7646089e77174b5ef6d720",
+ "301a91403585451cad67df96ada457b5",
+ "b2266a02a7154eb9a7a5ea36734a3902",
+ "cfb6680ece1946a990572948ecc15941",
+ "d851b02df22d40528579462ac2649e9c",
+ "bc75ab7805a041ad91ffcc6cf878212d",
+ "99ee702176d6453d9df40b1278a3dc0d",
+ "09a5640c9f9f4d1cb2d320e0f7577469"
+ ]
},
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "id": "SbyuZvarlToD"
- },
- "outputs": [],
- "source": [
- "%pip install transformers argilla datasets torch setfit -qqqqqqq"
- ]
+ "id": "ft7e8RApyW0u",
+ "outputId": "b7414e09-d2a5-4c8e-9dd2-364b53f8ee70"
+ },
+ "outputs": [],
+ "source": [
+ "zero_shot_news_dataset = [\n",
+ " rg.TextClassificationRecord(\n",
+ " text=res[\"sequence\"],\n",
+ " prediction=list(zip(res[\"labels\"], res[\"scores\"])),\n",
+ " annotation=record.prediction[0][0],\n",
+ " )\n",
+ " for res, record in zip(result, ds_record)\n",
+ "]\n",
+ "\n",
+ "rg.log(name=\"zero_shot_news_dataset\", records=zero_shot_news_dataset)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "T_weCa-YPAzF"
+ },
+ "source": [
+ "You can access the zero_shot_news_dataset in the Argilla UI:\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Z7GTj9I9K-mf"
+ },
+ "source": [
+ "Finally, we measure the performance of our model using the argilla f1 function that computes the F1 score:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 542
},
+ "id": "1iShwCbj3n-l",
+ "outputId": "b02aa4e4-cdc9-4d07-e2e0-f7d794b45706"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "XMxT5_tK346_"
- },
- "source": [
- "The imports needed:"
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "
\n",
+ "\n",
+ "2. [Launch Argilla using Argilla's quickstart Docker image](../../getting_started/quickstart.ipynb): This is the recommended option if you want Argilla running on your local machine. Note that this option will only let you run the tutorial locally and not with an external notebook service.\n",
+ "\n",
+ "For more information on deployment options, please check the Deployment section of the documentation.\n",
+ "\n",
+ "
\n",
+ "\n",
+ ">🤯 **Tip**\n",
+ "\n",
+ "> This tutorial is a Jupyter Notebook. There are two options to run it:\n",
+ "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
+ "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
+ "
\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "KBaIylKyh1mF"
+ },
+ "source": [
+ "## Setup\n",
+ "\n",
+ "For this tutorial, you'll need to install the Argilla client and a few third party libraries using `pip`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "80Xv9vqx0q4r",
+ "outputId": "1aa71f15-ed36-4feb-c909-5c7feaeb638e"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "10I7EROxWPqy"
- },
- "source": [
- ""
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m238.1/238.1 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.5/55.5 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.7/214.7 kB\u001b[0m \u001b[31m14.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m385.3/385.3 kB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.9/56.9 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.6/51.6 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m525.6/525.6 kB\u001b[0m \u001b[31m18.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.9/69.9 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m25.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.5/59.5 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.3/64.3 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.6/69.6 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.6/49.6 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.1/143.1 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m593.7/593.7 kB\u001b[0m \u001b[31m29.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.9/142.9 kB\u001b[0m \u001b[31m20.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.1/51.1 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m428.8/428.8 kB\u001b[0m \u001b[31m36.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.1/4.1 MB\u001b[0m \u001b[31m49.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m50.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hCollecting datasets\n",
+ " Downloading datasets-2.14.4-py3-none-any.whl (519 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.3/519.3 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.23.5)\n",
+ "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
+ "Collecting dill<0.3.8,>=0.3.0 (from datasets)\n",
+ " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
+ "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
+ "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.1)\n",
+ "Collecting xxhash (from datasets)\n",
+ " Downloading xxhash-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m22.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hCollecting multiprocess (from datasets)\n",
+ " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
+ "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.8.5)\n",
+ "Collecting huggingface-hub<1.0.0,>=0.14.0 (from datasets)\n",
+ " Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m35.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.1)\n",
+ "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
+ "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
+ "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (3.2.0)\n",
+ "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
+ "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
+ "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.2)\n",
+ "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n",
+ "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
+ "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.12.2)\n",
+ "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.7.1)\n",
+ "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.4)\n",
+ "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (1.26.16)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.7.22)\n",
+ "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
+ "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3)\n",
+ "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
+ "Installing collected packages: xxhash, dill, multiprocess, huggingface-hub, datasets\n",
+ "Successfully installed datasets-2.14.4 dill-0.3.7 huggingface-hub-0.16.4 multiprocess-0.70.15 xxhash-3.3.0\n",
+ "Collecting transformers\n",
+ " Downloading transformers-4.33.0-py3-none-any.whl (7.6 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
+ "Requirement already satisfied: huggingface-hub<1.0,>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.16.4)\n",
+ "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
+ "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
+ "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
+ "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
+ "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
+ "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
+ " Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m48.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
+ " Downloading safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m53.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
+ "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (2023.6.0)\n",
+ "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (4.7.1)\n",
+ "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.2.0)\n",
+ "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
+ "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.16)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
+ "Installing collected packages: tokenizers, safetensors, transformers\n",
+ "Successfully installed safetensors-0.3.3 tokenizers-0.13.3 transformers-4.33.0\n",
+ "Collecting evaluate\n",
+ " Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: datasets>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2.14.4)\n",
+ "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from evaluate) (1.23.5)\n",
+ "Requirement already satisfied: dill in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.3.7)\n",
+ "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from evaluate) (1.5.3)\n",
+ "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2.31.0)\n",
+ "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from evaluate) (4.66.1)\n",
+ "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from evaluate) (3.3.0)\n",
+ "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.70.15)\n",
+ "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2023.6.0)\n",
+ "Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.16.4)\n",
+ "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from evaluate) (23.1)\n",
+ "Collecting responses<0.19 (from evaluate)\n",
+ " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
+ "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (9.0.0)\n",
+ "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (3.8.5)\n",
+ "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (6.0.1)\n",
+ "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.7.0->evaluate) (3.12.2)\n",
+ "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.7.0->evaluate) (4.7.1)\n",
+ "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (3.2.0)\n",
+ "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (3.4)\n",
+ "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (1.26.16)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (2023.7.22)\n",
+ "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->evaluate) (2.8.2)\n",
+ "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->evaluate) (2023.3)\n",
+ "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.1.0)\n",
+ "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\n",
+ "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.3)\n",
+ "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.2)\n",
+ "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.0)\n",
+ "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n",
+ "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->evaluate) (1.16.0)\n",
+ "Installing collected packages: responses, evaluate\n",
+ "Successfully installed evaluate-0.4.0 responses-0.18.0\n",
+ "Collecting seqeval\n",
+ " Downloading seqeval-1.2.2.tar.gz (43 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+ "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.23.5)\n",
+ "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.2.2)\n",
+ "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.10.1)\n",
+ "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.3.2)\n",
+ "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (3.2.0)\n",
+ "Building wheels for collected packages: seqeval\n",
+ " Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
+ " Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16162 sha256=a3e4deed0ae4f82793ec07d332ea0faca9b72401ac85aa8047235d5fec9ef8ce\n",
+ " Stored in directory: /root/.cache/pip/wheels/1a/67/4a/ad4082dd7dfc30f2abfe4d80a2ed5926a506eb8a972b4767fa\n",
+ "Successfully built seqeval\n",
+ "Installing collected packages: seqeval\n",
+ "Successfully installed seqeval-1.2.2\n",
+ "Requirement already satisfied: transformers[torch] in /usr/local/lib/python3.10/dist-packages (4.33.0)\n",
+ "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (3.12.2)\n",
+ "Requirement already satisfied: huggingface-hub<1.0,>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.16.4)\n",
+ "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (1.23.5)\n",
+ "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (23.1)\n",
+ "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (6.0.1)\n",
+ "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2023.6.3)\n",
+ "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.31.0)\n",
+ "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.13.3)\n",
+ "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.3.3)\n",
+ "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (4.66.1)\n",
+ "Requirement already satisfied: torch!=1.12.0,>=1.10 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.0.1+cu118)\n",
+ "Collecting accelerate>=0.20.3 (from transformers[torch])\n",
+ " Downloading accelerate-0.22.0-py3-none-any.whl (251 kB)\n",
+ "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m251.2/251.2 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
+ "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.20.3->transformers[torch]) (5.9.5)\n",
+ "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers[torch]) (2023.6.0)\n",
+ "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers[torch]) (4.7.1)\n",
+ "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (1.12)\n",
+ "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (3.1)\n",
+ "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (3.1.2)\n",
+ "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (2.0.0)\n",
+ "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch!=1.12.0,>=1.10->transformers[torch]) (3.27.2)\n",
+ "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch!=1.12.0,>=1.10->transformers[torch]) (16.0.6)\n",
+ "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.2.0)\n",
+ "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.4)\n",
+ "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (1.26.16)\n",
+ "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2023.7.22)\n",
+ "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch!=1.12.0,>=1.10->transformers[torch]) (2.1.3)\n",
+ "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch!=1.12.0,>=1.10->transformers[torch]) (1.3.0)\n",
+ "Installing collected packages: accelerate\n",
+ "Successfully installed accelerate-0.22.0\n",
+ "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.22.0)\n",
+ "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.23.5)\n",
+ "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.1)\n",
+ "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
+ "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
+ "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.0.1+cu118)\n",
+ "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.12.2)\n",
+ "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.7.1)\n",
+ "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
+ "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1)\n",
+ "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
+ "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.0.0)\n",
+ "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (3.27.2)\n",
+ "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (16.0.6)\n",
+ "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
+ "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n"
+ ]
+ }
+ ],
+ "source": [
+ "%pip install \"argilla[server]==1.5.0\" -qqq\n",
+ "%pip install datasets\n",
+ "%pip install transformers\n",
+ "%pip install evaluate\n",
+ "%pip install seqeval\n",
+ "%pip install transformers[torch]\n",
+ "%pip install accelerate -U"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "PhoCuW7y0q4s"
+ },
+ "source": [
+ "Let's import the Argilla module for reading and writing data:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "metadata": {
+ "id": "bZ5IcIyG0q4t"
+ },
+ "outputs": [],
+ "source": [
+ "import argilla as rg"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "IO241Tyq0q4t"
+ },
+ "source": [
+ "If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "id": "5Olsh_SY0q4t"
+ },
+ "outputs": [],
+ "source": [
+ "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
+ "# Replace api_key if you configured a custom API key\n",
+ "# Replace workspace with the name of your workspace\n",
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"owner.apikey\", workspace=\"admin\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Set the HF_TOKEN environment variable\n",
+ "# import os\n",
+ "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
+ "\n",
+ "# # Replace api_url with the url to your HF Spaces URL\n",
+ "# # Replace api_key if you configured a custom API key\n",
+ "# # Replace workspace with the name of your workspace\n",
+ "# rg.init(\n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
+ "# api_key=\"owner.apikey\",\n",
+ "# workspace=\"admin\",\n",
+ "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
+ "# )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "LKgEhWk70q4u"
+ },
+ "source": [
+ "Finally, let's include the imports we need:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {
+ "id": "gW09SGfn0q4u"
+ },
+ "outputs": [],
+ "source": [
+ "import pandas as pd\n",
+ "import random\n",
+ "import evaluate\n",
+ "import transformers\n",
+ "import numpy as np\n",
+ "import torch\n",
+ "import pickle\n",
+ "\n",
+ "from datasets import load_dataset, ClassLabel, Sequence\n",
+ "from argilla.metrics.token_classification import top_k_mentions\n",
+ "from argilla.metrics.token_classification.metrics import Annotations\n",
+ "from IPython.display import display, HTML\n",
+ "from sklearn.model_selection import train_test_split\n",
+ "from transformers import (\n",
+ " AutoTokenizer,\n",
+ " AutoModelForTokenClassification,\n",
+ " TrainingArguments,\n",
+ " Trainer,\n",
+ " DataCollatorForTokenClassification,\n",
+ " pipeline,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Enable Telemetry\n",
+ "\n",
+ "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
+ " tutorial_running()\n",
+ "except ImportError:\n",
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "n40-m2PXmcM4"
+ },
+ "source": [
+ "## 🚀 Exploring our dataset"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "6KIAb_u4v8vL"
+ },
+ "source": [
+ "First, we will load the train of our dataset from HuggingFace in order to explore it using ``load_dataset``. And, as we can see, it has 119 entries and two columns: one with the sequence of tokens and the other with the sequence of NER tags."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 241,
+ "referenced_widgets": [
+ "99e5a771b1c94d74bf5c3e70e15ad09f",
+ "acf78f08f05043cdb3273d5c92c0df31",
+ "f0b531f225234f79b4ead2fad8fe2751",
+ "6f5f5d31a6f749158440b374c0950f64",
+ "00b9010d8b794a7594813f6894e5abfd",
+ "367c275138534c858c9313a36e154b10",
+ "7cb7ee5d7ee245e0a45520ba3a813f0c",
+ "7dc978907a3b435f98cfe63107929efd",
+ "a87aec084388404a904b5e214f60eacc",
+ "194082cd09ec44f2941f3f76134fb96c",
+ "c65d24e786c34452ad44f04fa87a5131",
+ "082f49ae0ab64400aedb4a8f88936285",
+ "df53731b5ee5449d905f0a8e9ed0979f",
+ "4c695ba86c6c469ea313a0080dea76bd",
+ "41057edf94144db5933b68ac821ae98a",
+ "1d2943840f294a57b1e3636555a4e25d",
+ "0f26cdb761b74c58b5002d3e592df69a",
+ "34c747c3411d48a5be4d7b858c7f1897",
+ "01e3c0d891154f3c8a94c9d2b5dd9bc9",
+ "150aa7a7933b44edb0110f53602fe1a9",
+ "b9d99330970c43db9a803bad08e87686",
+ "35b994f5411444a2a6800072017013df",
+ "a6739364524849378c94c89a72083871",
+ "9de0c934fe3c48169110a5e9c44cf67a",
+ "1f1ac3e94d9a482c863fcca0b3ffd976",
+ "fb746c052b884ddfa5d47ab3e925a790",
+ "3e15aa279b304b3a81176762a144efee",
+ "ca226c7464284dd8ac497a021cb2cd79",
+ "d8e4d43b5dac4b859fd77ab1c63cee7e",
+ "23e87cd64e544db290cee8a8e491aaa6",
+ "533d34aac5984fcdac3599d217a19269",
+ "0e1f107885b446e484e10d7c2bb3148d",
+ "d831eeae6fce4c1e952d3dddff336c44",
+ "ac97b8345ac449b6aeb13f3815ba614e",
+ "34209e5556f045d1a8dcd42799cd5b07",
+ "0913a4edc88e46e1bcb78a2eef6dbacd",
+ "a8491ebf827e4829911cdb6f129dbd9b",
+ "297762bdcf7a40da868faba7ba524f33",
+ "34eb03a9d5154fc5994a57a432241dbd",
+ "70ac8fda9db8482aa23d1412f1026303",
+ "902d902b20c449a4aec44d90c66a44d9",
+ "5434b0b2eebf4d56a4e1b54875582166",
+ "4784d47e7f1140e29e97b9232d95e39d",
+ "9db8cd674fa343c68c4cdea7728bc508",
+ "18d8e0b8ddeb43f098437dbe8de71d63",
+ "9b1f8961077d4cb895858f069b901d6b",
+ "6a624ed9e6ed4e2d88b05578d1f44060",
+ "97f3c530ddea4d2a92fd24a7bd1f36f4",
+ "5b42770f877d40f9bb9054a642de3b02",
+ "d19247d43efa4f5395479c0091908812",
+ "6a31b14c2db641a8881bca8a33b59c54",
+ "488568ad39a54008bdb9a4b6f09f7952",
+ "0b3e30198b7e49c581a5f438f23fce59",
+ "bac5002e328b47e29605b584e8f3d35c",
+ "70bcb268780e439098031084dff765e2",
+ "65fe01641cca423facc645cba13f35cd",
+ "9b067923d08f496cb2bad20ad7ef4697",
+ "366b9176dec742589ed7b423bf51b492",
+ "f7cbcbd2d4594c6bae7c1f9aaf5d9106",
+ "f2644660164f44b38a05172b51459ec1",
+ "1c1c5e928d4641ac93cec178438487c0",
+ "219a467d434248128c8e91cec11b7a16",
+ "fc092df95b51432c9836b3e0ba7234f1",
+ "130efa006fb44ef5948ee6ee4f66882f",
+ "1b4b6a31b84641308d77ab04fc85912e",
+ "3fd2f15677bc4cb9adc119c6158d0ec0",
+ "59a012d1a99a4aa4b88cc76e83787e56",
+ "203df8ab6cc948c69ed8469eac9de357",
+ "3243d051548c4996b10c970f582fe9f7",
+ "4ded40486ee64bdb874906972b9deec8",
+ "ffdadb62b0d74b1ea816cbb1bb924d2e",
+ "8412fa5c43c44b02ae2693e62705385c",
+ "9b7068fd51bb463e88e65803dbbbbade",
+ "a3d8351283934fdcb6c57470a12df1a1",
+ "4831180eb2214af081dfc1469f6c4021",
+ "63701b6fe5d34289bb9bb55da3aa8c49",
+ "00d55e9d9f08467e81a5b77fdaa35c07"
+ ]
},
+ "id": "s_e7D8paZFDq",
+ "outputId": "cbe1688f-b354-4205-aabd-edfa5dac9374"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "gs0Vhje60q4p"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "99e5a771b1c94d74bf5c3e70e15ad09f",
+ "version_major": 2,
+ "version_minor": 0
},
- "source": [
- "## Introduction\n",
- "\n",
- "Our goal is to show from a trainig dataset how to fine-tune a **tiny BERT model** in order to identify **NER** tags.\n",
- "\n",
- "For this purpose, we will first connect to Argilla and log our [dataset](https://huggingface.co/datasets/argilla/spacy_sm_wnut17), so that we can analyse it in a more visual way.\n",
- "\n",
- ">💡 **Tip:** If you want to try with a different dataset than the one in this tutorial, but it's not yet annotated, Argilla has several tutorials on how to do it [manually](/practical_guides/annotate_dataset.html) or [automatically](https://docs.v1.argilla.io/en/latest/tutorials/notebooks/labelling-tokenclassification-spacy-pretrained.html#Appendix:-Log-datasets-to-the-Hugging-Face-Hub).\n",
- "\n",
- "\n",
- "Next, we will preprocess our dataset and fine-tune the model. Here we will be using [DistilBERT](https://huggingface.co/docs/transformers/model_doc/distilbert), to make it easier to understand it and start *playing* with the parameters easily. However, there are still plenty of similar ones to [discover](https://huggingface.co/docs/transformers/index#bigtable).\n",
- "\n",
- "✨Let's get started!"
+ "text/plain": [
+ "Downloading readme: 0%| | 0.00/1.29k [00:00, ?B/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "markdown",
- "metadata": {
- "id": "WnKU83tp0q4q"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "082f49ae0ab64400aedb4a8f88936285",
+ "version_major": 2,
+ "version_minor": 0
},
- "source": [
- "## Running Argilla\n",
- "\n",
- "For this tutorial, you will need to have an Argilla server running. There are two main options for deploying and running Argilla:\n",
- "\n",
- "1. [Deploy Argilla on Hugging Face Spaces](https://huggingface.co/docs/hub/spaces-sdks-docker-argilla): This is the fastest option and the recommended choice for connecting to external notebooks (e.g., Google Colab) if you have an account on Hugging Face.\n",
- "\n",
- "
\n",
- "\n",
- "2. [Launch Argilla using Argilla's quickstart Docker image](../../getting_started/quickstart.ipynb): This is the recommended option if you want Argilla running on your local machine. Note that this option will only let you run the tutorial locally and not with an external notebook service.\n",
- "\n",
- "For more information on deployment options, please check the Deployment section of the documentation.\n",
- "\n",
- "
\n",
- "\n",
- ">🤯 **Tip**\n",
- "\n",
- "> This tutorial is a Jupyter Notebook. There are two options to run it:\n",
- "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
- "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
- "
\n"
+ "text/plain": [
+ "Downloading data files: 0%| | 0/2 [00:00, ?it/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "markdown",
- "metadata": {
- "id": "KBaIylKyh1mF"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "a6739364524849378c94c89a72083871",
+ "version_major": 2,
+ "version_minor": 0
},
- "source": [
- "## Setup\n",
- "\n",
- "For this tutorial, you'll need to install the Argilla client and a few third party libraries using `pip`:"
+ "text/plain": [
+ "Downloading data: 0%| | 0.00/14.2k [00:00, ?B/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "80Xv9vqx0q4r",
- "outputId": "1aa71f15-ed36-4feb-c909-5c7feaeb638e"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "ac97b8345ac449b6aeb13f3815ba614e",
+ "version_major": 2,
+ "version_minor": 0
},
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.0/2.0 MB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m71.5/71.5 kB\u001b[0m \u001b[31m8.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m238.1/238.1 kB\u001b[0m \u001b[31m15.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m55.5/55.5 kB\u001b[0m \u001b[31m7.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m214.7/214.7 kB\u001b[0m \u001b[31m14.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m385.3/385.3 kB\u001b[0m \u001b[31m17.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m56.9/56.9 kB\u001b[0m \u001b[31m5.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.6/51.6 kB\u001b[0m \u001b[31m7.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m45.7/45.7 kB\u001b[0m \u001b[31m6.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m525.6/525.6 kB\u001b[0m \u001b[31m18.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.9/69.9 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.7/2.7 MB\u001b[0m \u001b[31m25.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m59.5/59.5 kB\u001b[0m \u001b[31m8.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m3.1/3.1 MB\u001b[0m \u001b[31m33.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m64.3/64.3 kB\u001b[0m \u001b[31m8.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m69.6/69.6 kB\u001b[0m \u001b[31m9.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.6/49.6 kB\u001b[0m \u001b[31m6.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m143.1/143.1 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m593.7/593.7 kB\u001b[0m \u001b[31m29.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m142.9/142.9 kB\u001b[0m \u001b[31m20.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m51.1/51.1 kB\u001b[0m \u001b[31m6.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m8.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m428.8/428.8 kB\u001b[0m \u001b[31m36.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m4.1/4.1 MB\u001b[0m \u001b[31m49.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m50.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m129.9/129.9 kB\u001b[0m \u001b[31m16.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hCollecting datasets\n",
- " Downloading datasets-2.14.4-py3-none-any.whl (519 kB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m519.3/519.3 kB\u001b[0m \u001b[31m8.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hRequirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from datasets) (1.23.5)\n",
- "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (9.0.0)\n",
- "Collecting dill<0.3.8,>=0.3.0 (from datasets)\n",
- " Downloading dill-0.3.7-py3-none-any.whl (115 kB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m115.3/115.3 kB\u001b[0m \u001b[31m14.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hRequirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from datasets) (1.5.3)\n",
- "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from datasets) (2.31.0)\n",
- "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (4.66.1)\n",
- "Collecting xxhash (from datasets)\n",
- " Downloading xxhash-3.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (194 kB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m194.1/194.1 kB\u001b[0m \u001b[31m22.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hCollecting multiprocess (from datasets)\n",
- " Downloading multiprocess-0.70.15-py310-none-any.whl (134 kB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m134.8/134.8 kB\u001b[0m \u001b[31m18.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hRequirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (2023.6.0)\n",
- "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets) (3.8.5)\n",
- "Collecting huggingface-hub<1.0.0,>=0.14.0 (from datasets)\n",
- " Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m268.8/268.8 kB\u001b[0m \u001b[31m35.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hRequirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from datasets) (23.1)\n",
- "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets) (6.0.1)\n",
- "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (23.1.0)\n",
- "Requirement already satisfied: charset-normalizer<4.0,>=2.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (3.2.0)\n",
- "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (6.0.4)\n",
- "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (4.0.3)\n",
- "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.9.2)\n",
- "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.4.0)\n",
- "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets) (1.3.1)\n",
- "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (3.12.2)\n",
- "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0.0,>=0.14.0->datasets) (4.7.1)\n",
- "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (3.4)\n",
- "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (1.26.16)\n",
- "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->datasets) (2023.7.22)\n",
- "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2.8.2)\n",
- "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->datasets) (2023.3)\n",
- "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->datasets) (1.16.0)\n",
- "Installing collected packages: xxhash, dill, multiprocess, huggingface-hub, datasets\n",
- "Successfully installed datasets-2.14.4 dill-0.3.7 huggingface-hub-0.16.4 multiprocess-0.70.15 xxhash-3.3.0\n",
- "Collecting transformers\n",
- " Downloading transformers-4.33.0-py3-none-any.whl (7.6 MB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m19.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers) (3.12.2)\n",
- "Requirement already satisfied: huggingface-hub<1.0,>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (0.16.4)\n",
- "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (1.23.5)\n",
- "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers) (23.1)\n",
- "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers) (6.0.1)\n",
- "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers) (2023.6.3)\n",
- "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers) (2.31.0)\n",
- "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers)\n",
- " Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.8/7.8 MB\u001b[0m \u001b[31m48.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hCollecting safetensors>=0.3.1 (from transformers)\n",
- " Downloading safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.3/1.3 MB\u001b[0m \u001b[31m53.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers) (4.66.1)\n",
- "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (2023.6.0)\n",
- "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers) (4.7.1)\n",
- "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.2.0)\n",
- "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (3.4)\n",
- "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (1.26.16)\n",
- "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers) (2023.7.22)\n",
- "Installing collected packages: tokenizers, safetensors, transformers\n",
- "Successfully installed safetensors-0.3.3 tokenizers-0.13.3 transformers-4.33.0\n",
- "Collecting evaluate\n",
- " Downloading evaluate-0.4.0-py3-none-any.whl (81 kB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m81.4/81.4 kB\u001b[0m \u001b[31m2.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hRequirement already satisfied: datasets>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2.14.4)\n",
- "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from evaluate) (1.23.5)\n",
- "Requirement already satisfied: dill in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.3.7)\n",
- "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from evaluate) (1.5.3)\n",
- "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2.31.0)\n",
- "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.10/dist-packages (from evaluate) (4.66.1)\n",
- "Requirement already satisfied: xxhash in /usr/local/lib/python3.10/dist-packages (from evaluate) (3.3.0)\n",
- "Requirement already satisfied: multiprocess in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.70.15)\n",
- "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (2023.6.0)\n",
- "Requirement already satisfied: huggingface-hub>=0.7.0 in /usr/local/lib/python3.10/dist-packages (from evaluate) (0.16.4)\n",
- "Requirement already satisfied: packaging in /usr/local/lib/python3.10/dist-packages (from evaluate) (23.1)\n",
- "Collecting responses<0.19 (from evaluate)\n",
- " Downloading responses-0.18.0-py3-none-any.whl (38 kB)\n",
- "Requirement already satisfied: pyarrow>=8.0.0 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (9.0.0)\n",
- "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (3.8.5)\n",
- "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from datasets>=2.0.0->evaluate) (6.0.1)\n",
- "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.7.0->evaluate) (3.12.2)\n",
- "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.7.0->evaluate) (4.7.1)\n",
- "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (3.2.0)\n",
- "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (3.4)\n",
- "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (1.26.16)\n",
- "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests>=2.19.0->evaluate) (2023.7.22)\n",
- "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->evaluate) (2.8.2)\n",
- "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->evaluate) (2023.3)\n",
- "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (23.1.0)\n",
- "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (6.0.4)\n",
- "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (4.0.3)\n",
- "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.9.2)\n",
- "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.4.0)\n",
- "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->datasets>=2.0.0->evaluate) (1.3.1)\n",
- "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from python-dateutil>=2.8.1->pandas->evaluate) (1.16.0)\n",
- "Installing collected packages: responses, evaluate\n",
- "Successfully installed evaluate-0.4.0 responses-0.18.0\n",
- "Collecting seqeval\n",
- " Downloading seqeval-1.2.2.tar.gz (43 kB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m43.6/43.6 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25h Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
- "Requirement already satisfied: numpy>=1.14.0 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.23.5)\n",
- "Requirement already satisfied: scikit-learn>=0.21.3 in /usr/local/lib/python3.10/dist-packages (from seqeval) (1.2.2)\n",
- "Requirement already satisfied: scipy>=1.3.2 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.10.1)\n",
- "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (1.3.2)\n",
- "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=0.21.3->seqeval) (3.2.0)\n",
- "Building wheels for collected packages: seqeval\n",
- " Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
- " Created wheel for seqeval: filename=seqeval-1.2.2-py3-none-any.whl size=16162 sha256=a3e4deed0ae4f82793ec07d332ea0faca9b72401ac85aa8047235d5fec9ef8ce\n",
- " Stored in directory: /root/.cache/pip/wheels/1a/67/4a/ad4082dd7dfc30f2abfe4d80a2ed5926a506eb8a972b4767fa\n",
- "Successfully built seqeval\n",
- "Installing collected packages: seqeval\n",
- "Successfully installed seqeval-1.2.2\n",
- "Requirement already satisfied: transformers[torch] in /usr/local/lib/python3.10/dist-packages (4.33.0)\n",
- "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (3.12.2)\n",
- "Requirement already satisfied: huggingface-hub<1.0,>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.16.4)\n",
- "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (1.23.5)\n",
- "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (23.1)\n",
- "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (6.0.1)\n",
- "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2023.6.3)\n",
- "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.31.0)\n",
- "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.13.3)\n",
- "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (0.3.3)\n",
- "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (4.66.1)\n",
- "Requirement already satisfied: torch!=1.12.0,>=1.10 in /usr/local/lib/python3.10/dist-packages (from transformers[torch]) (2.0.1+cu118)\n",
- "Collecting accelerate>=0.20.3 (from transformers[torch])\n",
- " Downloading accelerate-0.22.0-py3-none-any.whl (251 kB)\n",
- "\u001b[2K \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m251.2/251.2 kB\u001b[0m \u001b[31m5.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
- "\u001b[?25hRequirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.20.3->transformers[torch]) (5.9.5)\n",
- "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers[torch]) (2023.6.0)\n",
- "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers[torch]) (4.7.1)\n",
- "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (1.12)\n",
- "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (3.1)\n",
- "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (3.1.2)\n",
- "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.10->transformers[torch]) (2.0.0)\n",
- "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch!=1.12.0,>=1.10->transformers[torch]) (3.27.2)\n",
- "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch!=1.12.0,>=1.10->transformers[torch]) (16.0.6)\n",
- "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.2.0)\n",
- "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (3.4)\n",
- "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (1.26.16)\n",
- "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->transformers[torch]) (2023.7.22)\n",
- "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch!=1.12.0,>=1.10->transformers[torch]) (2.1.3)\n",
- "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch!=1.12.0,>=1.10->transformers[torch]) (1.3.0)\n",
- "Installing collected packages: accelerate\n",
- "Successfully installed accelerate-0.22.0\n",
- "Requirement already satisfied: accelerate in /usr/local/lib/python3.10/dist-packages (0.22.0)\n",
- "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from accelerate) (1.23.5)\n",
- "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (23.1)\n",
- "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate) (5.9.5)\n",
- "Requirement already satisfied: pyyaml in /usr/local/lib/python3.10/dist-packages (from accelerate) (6.0.1)\n",
- "Requirement already satisfied: torch>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from accelerate) (2.0.1+cu118)\n",
- "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.12.2)\n",
- "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (4.7.1)\n",
- "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (1.12)\n",
- "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1)\n",
- "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (3.1.2)\n",
- "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch>=1.10.0->accelerate) (2.0.0)\n",
- "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (3.27.2)\n",
- "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch>=1.10.0->accelerate) (16.0.6)\n",
- "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch>=1.10.0->accelerate) (2.1.3)\n",
- "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch>=1.10.0->accelerate) (1.3.0)\n"
- ]
- }
- ],
- "source": [
- "%pip install \"argilla[server]==1.5.0\" -qqq\n",
- "%pip install datasets\n",
- "%pip install transformers\n",
- "%pip install evaluate\n",
- "%pip install seqeval\n",
- "%pip install transformers[torch]\n",
- "%pip install accelerate -U"
+ "text/plain": [
+ "Downloading data: 0%| | 0.00/5.03k [00:00, ?B/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "markdown",
- "metadata": {
- "id": "PhoCuW7y0q4s"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "18d8e0b8ddeb43f098437dbe8de71d63",
+ "version_major": 2,
+ "version_minor": 0
},
- "source": [
- "Let's import the Argilla module for reading and writing data:"
+ "text/plain": [
+ "Extracting data files: 0%| | 0/2 [00:00, ?it/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "code",
- "execution_count": 2,
- "metadata": {
- "id": "bZ5IcIyG0q4t"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "65fe01641cca423facc645cba13f35cd",
+ "version_major": 2,
+ "version_minor": 0
},
- "outputs": [],
- "source": [
- "import argilla as rg"
+ "text/plain": [
+ "Generating train split: 0%| | 0/119 [00:00, ? examples/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "markdown",
- "metadata": {
- "id": "IO241Tyq0q4t"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "59a012d1a99a4aa4b88cc76e83787e56",
+ "version_major": 2,
+ "version_minor": 0
},
- "source": [
- "If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
+ "text/plain": [
+ "Generating test split: 0%| | 0/30 [00:00, ? examples/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "dataset = load_dataset(\"argilla/spacy_sm_wnut17\", split=\"train\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "L9_dS8-SvAPP",
+ "outputId": "2187d913-c167-4089-f4d5-933b0f07e71a"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {
- "id": "5Olsh_SY0q4t"
- },
- "outputs": [],
- "source": [
- "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
- "# Replace api_key if you configured a custom API key\n",
- "# Replace workspace with the name of your workspace\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"owner.apikey\",\n",
- " workspace=\"admin\"\n",
- ")"
+ "data": {
+ "text/plain": [
+ "Dataset({\n",
+ " features: ['tokens', 'ner_tags'],\n",
+ " num_rows: 119\n",
+ "})"
]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "dataset"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ITgB-mzZz3vX"
+ },
+ "source": [
+ "Next, we will use the following code taking advantage of the ``DatasetDict`` option ``Features`` to convert it to the format required by Argilla in order to log it.\n",
+ "\n",
+ "The three elements that our data must have for Token Classifications are the following:\n",
+ "\n",
+ "* **text**: the complete string.\n",
+ "* **tokens**: the sequence of tokens.\n",
+ "* **annotation**: a tuple formed by the tag, the start position and the end position.\n",
+ "\n",
+ "> ⚠️ **Be careful:** Each execution will upload and add your annotations again without being overwritten."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 79,
+ "metadata": {
+ "id": "K7WFV0vaHhyd"
+ },
+ "outputs": [],
+ "source": [
+ "# Create a function to read the sequences\n",
+ "def parse_entities(record):\n",
+ " current_entity = None # to check if current entity in process\n",
+ " current_info = [] # to save the information used in the tuple for the whole sentence\n",
+ " char_position = 0\n",
+ " entities = [] # final list to save the tuples\n",
+ "\n",
+ " # Iterate over the tokens and ner tags\n",
+ " for i in range(len(record[\"ner_tags\"])):\n",
+ " token = record[\"tokens\"][i]\n",
+ " ner_tag = dataset.features[\"ner_tags\"].feature.names[record[\"ner_tags\"][i]]\n",
+ "\n",
+ " if ner_tag.startswith(\"B-\"):\n",
+ " if current_entity:\n",
+ " current_info.append(current_entity)\n",
+ " current_entity = {\"word\": token, \"start\": char_position, \"tag\": ner_tag[2:]}\n",
+ " char_position += len(token) + 1\n",
+ "\n",
+ " elif ner_tag.startswith(\"I-\"):\n",
+ " if current_entity:\n",
+ " current_entity[\"word\"] += \" \" + token\n",
+ " char_position += len(token) + 1\n",
+ "\n",
+ " elif ner_tag == \"O\":\n",
+ " char_position += len(token) + 1\n",
+ "\n",
+ " # Add the last entity if it exists\n",
+ " if current_entity:\n",
+ " current_info.append(current_entity)\n",
+ "\n",
+ " # Calculate the end positions for each entity\n",
+ " for entity in current_info:\n",
+ " entity[\"end\"] = entity[\"start\"] + len(entity[\"word\"])\n",
+ "\n",
+ " for entity in current_info:\n",
+ " entities.append((entity[\"tag\"], entity[\"start\"], entity[\"end\"]))\n",
+ "\n",
+ " return entities"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "WsKBFFAiYv3D"
+ },
+ "outputs": [],
+ "source": [
+ "# Write a loop to iterate over each row of your dataset and add the text, tokens, and tuple\n",
+ "records = [\n",
+ " rg.TokenClassificationRecord(\n",
+ " text=\" \".join(row[\"tokens\"]),\n",
+ " tokens=row[\"tokens\"],\n",
+ " annotation=parse_entities(row),\n",
+ " )\n",
+ " for row in dataset\n",
+ "]\n",
+ "\n",
+ "# Log the records with the name of your choice\n",
+ "rg.log(records, \"spacy_sm_wnut17\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "3Ydx5se82s_n"
+ },
+ "source": [
+ "So now you will be able to check your annotations in a much more visual way and even edit them if necessary.\n",
+ "\n",
+ "\n",
+ "\n",
+ "In addition, **Argilla** also has more options, e.g. to extract [metrics](https://docs.v1.argilla.io/en/latest/reference/python/python_metrics.html) such as the one shown below.\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 542
},
+ "id": "uIg23A-uRX94",
+ "outputId": "0a9fed8e-6ae8-4a96-c40f-420f61f55c73"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "\n",
+ "
\n",
+ "
\n",
+ "\n",
+ ""
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Select the dataset from Argilla and visualize the data\n",
+ "top_k_mentions(\n",
+ " name=\"spacy_sm_wnut17\", k=30, threshold=2, compute_for=Annotations\n",
+ ").visualize()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ypCRJhX35ltp"
+ },
+ "source": [
+ "## ⏳ Preprocessing the data"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ffpxCD1M8mFO"
+ },
+ "source": [
+ "Next, we will **pre-process our data** in the required format so that the model can work with it. In our case, we will reload them from HuggingFace, as in Argilla we only loaded the train set, however, this is also possible.\n",
+ "\n",
+ "The following code would allow us to prepare our data using Argilla, this is especially useful for manual annotations, as it adds **B-** (beggining) or **I-** (inside) to our NER tags automatically depending on their position.\n",
+ "\n",
+ "```python\n",
+ "dataset = rg.load(\"dataset_name\").prepare_for_training()\n",
+ "\n",
+ "dataset = dataset.train_test_split()\n",
+ "```\n",
+ "\n",
+ "> 🤯 **Tip:** In our case, we are working with a very small dataset that is divided into train and test. However, you may are using another dataset that already have the ``validation`` partition, or even if it is larger, you can create this partition yourself with the following code:\n",
+ "\n",
+ "```python\n",
+ "dataset['train'], dataset['validation'] = dataset['train'].train_test_split(.1).values()\n",
+ "```\n",
+ "So, let's continue!"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 240,
+ "referenced_widgets": [
+ "368c173b5ae942aca2136dfe7968e527",
+ "b27a1ac0f80443a18f51002768ef5883",
+ "ba728a77e57243e4b6a58c09d1d68c19",
+ "84cf7979444c46409f547fc34fe5959f",
+ "25731b7b5adf4f62953468b72d4dc746",
+ "c7c4843952414605b4a3a526ff32ad4f",
+ "215f5ed3a7ab450d9bfc87d0e92c306d",
+ "a0d5d3e4e2b24345aa002b6fdcd961a1",
+ "3a2ce5219b5349be87932029056fb270",
+ "eda3a7954d6a49418962197d7b120072",
+ "3732ff34dde142e1a934ef62b4f2f4f9"
+ ]
},
+ "id": "pl1MwBhxkQEm",
+ "outputId": "7296bdd5-652d-477b-9696-cee344341902"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# # Set the HF_TOKEN environment variable\n",
- "# import os\n",
- "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
- "\n",
- "# # Replace api_url with the url to your HF Spaces URL\n",
- "# # Replace api_key if you configured a custom API key\n",
- "# # Replace workspace with the name of your workspace\n",
- "# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
- "# api_key=\"owner.apikey\",\n",
- "# workspace=\"admin\",\n",
- "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
- "# )"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "WARNING:datasets.builder:Found cached dataset parquet (/root/.cache/huggingface/datasets/argilla___parquet/argilla--spacy_sm_wnut17-1babd564207f27f8/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7)\n"
+ ]
},
{
- "cell_type": "markdown",
- "metadata": {
- "id": "LKgEhWk70q4u"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "368c173b5ae942aca2136dfe7968e527",
+ "version_major": 2,
+ "version_minor": 0
},
- "source": [
- "Finally, let's include the imports we need:"
+ "text/plain": [
+ " 0%| | 0/2 [00:00, ?it/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {
- "id": "gW09SGfn0q4u"
- },
- "outputs": [],
- "source": [
- "import pandas as pd\n",
- "import random\n",
- "import evaluate\n",
- "import transformers\n",
- "import numpy as np\n",
- "import torch\n",
- "import pickle\n",
- "\n",
- "from datasets import load_dataset, ClassLabel, Sequence\n",
- "from argilla.metrics.token_classification import top_k_mentions\n",
- "from argilla.metrics.token_classification.metrics import Annotations\n",
- "from IPython.display import display, HTML\n",
- "from sklearn.model_selection import train_test_split\n",
- "from transformers import AutoTokenizer, AutoModelForTokenClassification, TrainingArguments, Trainer, DataCollatorForTokenClassification, pipeline"
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "DatasetDict({\n",
+ " train: Dataset({\n",
+ " features: ['tokens', 'ner_tags'],\n",
+ " num_rows: 119\n",
+ " })\n",
+ " test: Dataset({\n",
+ " features: ['tokens', 'ner_tags'],\n",
+ " num_rows: 30\n",
+ " })\n",
+ "})\n"
+ ]
+ }
+ ],
+ "source": [
+ "dataset = load_dataset(\"argilla/spacy_sm_wnut17\")\n",
+ "print(dataset)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "jeAuJM2XAX11"
+ },
+ "source": [
+ "Time to **tokenize**! Although it seems like this is already done, each token still needs to be converted into a vector (ID) that the model can read from its pretrained vocabulary. To do this, we will use ``AutoTokenizer.from_pretrained`` and the FastTokenizer ``distilbert-base-uncased``."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "eXNLu_-nIrJI"
+ },
+ "outputs": [],
+ "source": [
+ "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
+ "assert isinstance(tokenizer, transformers.PreTrainedTokenizerFast)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "PRzo_3NldUew",
+ "outputId": "7214186b-c584-4843-f10b-7144983ae402"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Enable Telemetry\n",
- "\n",
- "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
- ]
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "['says', 'it', \"'s\", 'Saturday', '!', 'I', \"'m\", 'wearing', 'my', 'Weekend', '!', ':)']\n",
+ "['[CLS]', 'says', 'it', \"'\", 's', 'saturday', '!', 'i', \"'\", 'm', 'wearing', 'my', 'weekend', '!', ':', ')', '[SEP]']\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Example of original tokens\n",
+ "example = dataset[\"train\"][0]\n",
+ "print(example[\"tokens\"])\n",
+ "\n",
+ "# Example after executing the AutoTokenizer\n",
+ "tokenized_input = tokenizer(example[\"tokens\"], is_split_into_words=True)\n",
+ "tokens = tokenizer.convert_ids_to_tokens(tokenized_input[\"input_ids\"])\n",
+ "print(tokens)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "Qzw7BRWYGiZn"
+ },
+ "source": [
+ "However, we now ran into a new problem. Since it tokenises according to a pre-trained vocabulary, this will create new subdivisions in some words (e.g. \"'s\" in \"'\" and \"s\"). In addition, it adds two new tags [CLS] and [SEP]. Therefore, we have to realign the IDs of our words with the corresponding NER tags, thanks to the ``word-ids`` method."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 54,
+ "referenced_widgets": [
+ "011e772f270e4e3b819fc093928b36b7",
+ "76ac206691ad4795b168534e69ed5a56",
+ "ac140d48d0984ca59e5e4fc437eff946",
+ "527332d8360b4df293452d0fcced938d",
+ "25b47d1f214340baa662328b090f2b6a",
+ "eeb298e8c2d74439950325d4e01fc0c2",
+ "f148e78f04274ae9a6dca98e4cf6c832",
+ "d22c976637264867b789a32857ffc717",
+ "cf6968f163624bb49e9f2fe5fb557b2d",
+ "b503a56aec3a4b09ae141e0af4adac23",
+ "7651cd2a101248f59a4216ef50027e90"
+ ]
},
+ "id": "vc0BSBLIIrJQ",
+ "outputId": "32780956-b8a8-4fdd-e296-0f85045c525b"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "try:\n",
- " from argilla.utils.telemetry import tutorial_running\n",
- " tutorial_running()\n",
- "except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "WARNING:datasets.arrow_dataset:Loading cached processed dataset at /root/.cache/huggingface/datasets/argilla___parquet/argilla--spacy_sm_wnut17-1babd564207f27f8/0.0.0/14a00e99c0d15a23649d0db8944380ac81082d4b021f398733dd84f3a6c569a7/cache-55b667584ffacf49.arrow\n"
+ ]
},
{
- "cell_type": "markdown",
- "metadata": {
- "id": "n40-m2PXmcM4"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "011e772f270e4e3b819fc093928b36b7",
+ "version_major": 2,
+ "version_minor": 0
},
- "source": [
- "## 🚀 Exploring our dataset"
+ "text/plain": [
+ "Map: 0%| | 0/30 [00:00, ? examples/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "label_all_tokens = True\n",
+ "\n",
+ "\n",
+ "def tokenize_and_align_labels(examples):\n",
+ " tokenized_inputs = tokenizer(\n",
+ " examples[\"tokens\"], truncation=True, is_split_into_words=True\n",
+ " )\n",
+ "\n",
+ " labels = []\n",
+ " for i, label in enumerate(examples[\"ner_tags\"]):\n",
+ " word_ids = tokenized_inputs.word_ids(batch_index=i)\n",
+ " previous_word_idx = None\n",
+ " label_ids = []\n",
+ " for word_idx in word_ids:\n",
+ " # Special tokens have a word id that is None. We set the label to -100 so they are automatically\n",
+ " # ignored in the loss function.\n",
+ " if word_idx is None:\n",
+ " label_ids.append(-100)\n",
+ " # We set the label for the first token of each word.\n",
+ " elif word_idx != previous_word_idx:\n",
+ " label_ids.append(label[word_idx])\n",
+ " # For the other tokens in a word, we set the label to either the current label or -100, depending on\n",
+ " # the label_all_tokens flag.\n",
+ " else:\n",
+ " label_ids.append(label[word_idx] if label_all_tokens else -100)\n",
+ " previous_word_idx = word_idx\n",
+ "\n",
+ " labels.append(label_ids)\n",
+ "\n",
+ " tokenized_inputs[\"labels\"] = labels\n",
+ " return tokenized_inputs\n",
+ "\n",
+ "\n",
+ "tokenized_dataset = dataset.map(tokenize_and_align_labels, batched=True)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "4HttMIC1KWWv"
+ },
+ "source": [
+ "## 🔍 Fine-tunning the model\n",
+ "\n",
+ "We should now start preparing the parameters of our model, i.e. we should start fine-tuning."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "UauakSYrMs3d"
+ },
+ "source": [
+ "### Model\n",
+ "\n",
+ "First we will download our pretrained model with ``AutoModelForTokenClassification`` and we will indicate the name of the chosen model, the number of tags and we will indicate the correspondences between their IDs and their names.\n",
+ "\n",
+ "In addition, we will also set our ``DataCollator`` to form a batch by using our processed examples as input. In this case, we will be using ``DataCollatorForTokenClassification``."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
+ "id": "M9CW8u2mNKmG",
+ "outputId": "ad32132a-ee47-49e9-b740-441d88c9b7f5"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {
- "id": "6KIAb_u4v8vL"
- },
- "source": [
- "First, we will load the train of our dataset from HuggingFace in order to explore it using ``load_dataset``. And, as we can see, it has 119 entries and two columns: one with the sequence of tokens and the other with the sequence of NER tags."
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForTokenClassification: ['vocab_transform.bias', 'vocab_projector.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_transform.weight']\n",
+ "- This IS expected if you are initializing DistilBertForTokenClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
+ "- This IS NOT expected if you are initializing DistilBertForTokenClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
+ "Some weights of DistilBertForTokenClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
+ "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Create a dictionary with the ids and the relevant label.\n",
+ "label_list = dataset[\"train\"].features[\"ner_tags\"].feature.names\n",
+ "id2label = {i: label for i, label in enumerate(label_list)}\n",
+ "label2id = {v: k for k, v in id2label.items()}\n",
+ "\n",
+ "# Download the model.\n",
+ "model = AutoModelForTokenClassification.from_pretrained(\n",
+ " \"distilbert-base-uncased\",\n",
+ " num_labels=len(label_list),\n",
+ " id2label=id2label,\n",
+ " label2id=label2id,\n",
+ ")\n",
+ "\n",
+ "# Set the DataCollator\n",
+ "data_collator = DataCollatorForTokenClassification(tokenizer)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "mkTNDfJWN0PH"
+ },
+ "source": [
+ "### Training arguments\n",
+ "The ``TrainingArguments`` class will include the [parameters](https://huggingface.co/docs/transformers/main_classes/trainer#transformers.TrainingArguments) to customise our training.\n",
+ "\n",
+ "> 💡 **Tip:** If you are using HuggingFace it may be easier for you to save your model there directly. To do so, use the following code and add the following parameters to TrainingArguments.\n",
+ "\n",
+ "```python\n",
+ "from huggingface_hub import notebook_login\n",
+ "notebook_login()\n",
+ "\n",
+ "# Add the following parameter\n",
+ "training_args = TrainingArguments(\n",
+ " save_strategy=\"epoch\",\n",
+ " load_best_model_at_end=True,\n",
+ " push_to_hub=True,\n",
+ ")\n",
+ "```\n",
+ "> 🕹️ **Let's play:** What is the best accuracy you can get?"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "O9lz1LbVN3OW"
+ },
+ "outputs": [],
+ "source": [
+ "training_args = TrainingArguments(\n",
+ " output_dir=\"ner-recognition\",\n",
+ " learning_rate=2e-4,\n",
+ " per_device_train_batch_size=32,\n",
+ " per_device_eval_batch_size=32,\n",
+ " num_train_epochs=20,\n",
+ " weight_decay=0.05,\n",
+ " evaluation_strategy=\"epoch\",\n",
+ " optim=\"adamw_torch\",\n",
+ " logging_steps=50,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "7SFjM6KGN3s3"
+ },
+ "source": [
+ "### Metrics\n",
+ "\n",
+ "To know how our training has gone, of course, we must use metrics. Therefore, we will use ``Seqeval`` and a function that computes precision, recall, F1 and accuracy from the actual and predicted tags."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 49,
+ "referenced_widgets": [
+ "93d7c54a35494535a500550d73d22acb",
+ "ea8acd336a384c8793651542d6bb5167",
+ "9867f7c33e24416faf662ec5366af2f1",
+ "e023fa533e1a4059b8087be03f81ac9a",
+ "de682ca9943648579f74c077dec8bcfc",
+ "75db171714734f49b05273ba2d42be11",
+ "1ced5374042343c48b7523e0db3b05ec",
+ "9e837cd94af54d01a4ba079f16b881bf",
+ "3a0a83c63061434887f2b9760fdf5281",
+ "a309e1df709042d2a3e31df5f3426367",
+ "fe27af60738749bda14625b756362aef"
+ ]
},
+ "id": "Jr559HdeN6Q9",
+ "outputId": "dac3b0d0-173a-4b70-a2cf-d4306eecfd6e"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 241,
- "referenced_widgets": [
- "99e5a771b1c94d74bf5c3e70e15ad09f",
- "acf78f08f05043cdb3273d5c92c0df31",
- "f0b531f225234f79b4ead2fad8fe2751",
- "6f5f5d31a6f749158440b374c0950f64",
- "00b9010d8b794a7594813f6894e5abfd",
- "367c275138534c858c9313a36e154b10",
- "7cb7ee5d7ee245e0a45520ba3a813f0c",
- "7dc978907a3b435f98cfe63107929efd",
- "a87aec084388404a904b5e214f60eacc",
- "194082cd09ec44f2941f3f76134fb96c",
- "c65d24e786c34452ad44f04fa87a5131",
- "082f49ae0ab64400aedb4a8f88936285",
- "df53731b5ee5449d905f0a8e9ed0979f",
- "4c695ba86c6c469ea313a0080dea76bd",
- "41057edf94144db5933b68ac821ae98a",
- "1d2943840f294a57b1e3636555a4e25d",
- "0f26cdb761b74c58b5002d3e592df69a",
- "34c747c3411d48a5be4d7b858c7f1897",
- "01e3c0d891154f3c8a94c9d2b5dd9bc9",
- "150aa7a7933b44edb0110f53602fe1a9",
- "b9d99330970c43db9a803bad08e87686",
- "35b994f5411444a2a6800072017013df",
- "a6739364524849378c94c89a72083871",
- "9de0c934fe3c48169110a5e9c44cf67a",
- "1f1ac3e94d9a482c863fcca0b3ffd976",
- "fb746c052b884ddfa5d47ab3e925a790",
- "3e15aa279b304b3a81176762a144efee",
- "ca226c7464284dd8ac497a021cb2cd79",
- "d8e4d43b5dac4b859fd77ab1c63cee7e",
- "23e87cd64e544db290cee8a8e491aaa6",
- "533d34aac5984fcdac3599d217a19269",
- "0e1f107885b446e484e10d7c2bb3148d",
- "d831eeae6fce4c1e952d3dddff336c44",
- "ac97b8345ac449b6aeb13f3815ba614e",
- "34209e5556f045d1a8dcd42799cd5b07",
- "0913a4edc88e46e1bcb78a2eef6dbacd",
- "a8491ebf827e4829911cdb6f129dbd9b",
- "297762bdcf7a40da868faba7ba524f33",
- "34eb03a9d5154fc5994a57a432241dbd",
- "70ac8fda9db8482aa23d1412f1026303",
- "902d902b20c449a4aec44d90c66a44d9",
- "5434b0b2eebf4d56a4e1b54875582166",
- "4784d47e7f1140e29e97b9232d95e39d",
- "9db8cd674fa343c68c4cdea7728bc508",
- "18d8e0b8ddeb43f098437dbe8de71d63",
- "9b1f8961077d4cb895858f069b901d6b",
- "6a624ed9e6ed4e2d88b05578d1f44060",
- "97f3c530ddea4d2a92fd24a7bd1f36f4",
- "5b42770f877d40f9bb9054a642de3b02",
- "d19247d43efa4f5395479c0091908812",
- "6a31b14c2db641a8881bca8a33b59c54",
- "488568ad39a54008bdb9a4b6f09f7952",
- "0b3e30198b7e49c581a5f438f23fce59",
- "bac5002e328b47e29605b584e8f3d35c",
- "70bcb268780e439098031084dff765e2",
- "65fe01641cca423facc645cba13f35cd",
- "9b067923d08f496cb2bad20ad7ef4697",
- "366b9176dec742589ed7b423bf51b492",
- "f7cbcbd2d4594c6bae7c1f9aaf5d9106",
- "f2644660164f44b38a05172b51459ec1",
- "1c1c5e928d4641ac93cec178438487c0",
- "219a467d434248128c8e91cec11b7a16",
- "fc092df95b51432c9836b3e0ba7234f1",
- "130efa006fb44ef5948ee6ee4f66882f",
- "1b4b6a31b84641308d77ab04fc85912e",
- "3fd2f15677bc4cb9adc119c6158d0ec0",
- "59a012d1a99a4aa4b88cc76e83787e56",
- "203df8ab6cc948c69ed8469eac9de357",
- "3243d051548c4996b10c970f582fe9f7",
- "4ded40486ee64bdb874906972b9deec8",
- "ffdadb62b0d74b1ea816cbb1bb924d2e",
- "8412fa5c43c44b02ae2693e62705385c",
- "9b7068fd51bb463e88e65803dbbbbade",
- "a3d8351283934fdcb6c57470a12df1a1",
- "4831180eb2214af081dfc1469f6c4021",
- "63701b6fe5d34289bb9bb55da3aa8c49",
- "00d55e9d9f08467e81a5b77fdaa35c07"
- ]
- },
- "id": "s_e7D8paZFDq",
- "outputId": "cbe1688f-b354-4205-aabd-edfa5dac9374"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "93d7c54a35494535a500550d73d22acb",
+ "version_major": 2,
+ "version_minor": 0
},
- "outputs": [
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "99e5a771b1c94d74bf5c3e70e15ad09f",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Downloading readme: 0%| | 0.00/1.29k [00:00, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "082f49ae0ab64400aedb4a8f88936285",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Downloading data files: 0%| | 0/2 [00:00, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "a6739364524849378c94c89a72083871",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Downloading data: 0%| | 0.00/14.2k [00:00, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "ac97b8345ac449b6aeb13f3815ba614e",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Downloading data: 0%| | 0.00/5.03k [00:00, ?B/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "18d8e0b8ddeb43f098437dbe8de71d63",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Extracting data files: 0%| | 0/2 [00:00, ?it/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "65fe01641cca423facc645cba13f35cd",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Generating train split: 0%| | 0/119 [00:00, ? examples/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- },
- {
- "data": {
- "application/vnd.jupyter.widget-view+json": {
- "model_id": "59a012d1a99a4aa4b88cc76e83787e56",
- "version_major": 2,
- "version_minor": 0
- },
- "text/plain": [
- "Generating test split: 0%| | 0/30 [00:00, ? examples/s]"
- ]
- },
- "metadata": {},
- "output_type": "display_data"
- }
- ],
- "source": [
- "dataset = load_dataset(\"argilla/spacy_sm_wnut17\", split = \"train\")"
+ "text/plain": [
+ "Downloading builder script: 0%| | 0.00/6.34k [00:00, ?B/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "# Load Sqeval.\n",
+ "metric = evaluate.load(\"seqeval\")\n",
+ "\n",
+ "# Create the list with the tags.\n",
+ "labels = [label_list[i] for i in example[f\"ner_tags\"]]\n",
+ "\n",
+ "\n",
+ "# Function to compute precision, recall, F1 and accuracy.\n",
+ "def compute_metrics(p):\n",
+ " predictions, labels = p\n",
+ " predictions = np.argmax(predictions, axis=2)\n",
+ "\n",
+ " true_predictions = [\n",
+ " [label_list[p] for (p, l) in zip(prediction, label) if l != -100]\n",
+ " for prediction, label in zip(predictions, labels)\n",
+ " ]\n",
+ " true_labels = [\n",
+ " [label_list[l] for (p, l) in zip(prediction, label) if l != -100]\n",
+ " for prediction, label in zip(predictions, labels)\n",
+ " ]\n",
+ "\n",
+ " results = metric.compute(predictions=true_predictions, references=true_labels)\n",
+ " return {\n",
+ " \"precision\": results[\"overall_precision\"],\n",
+ " \"recall\": results[\"overall_recall\"],\n",
+ " \"f1\": results[\"overall_f1\"],\n",
+ " \"accuracy\": results[\"overall_accuracy\"],\n",
+ " }"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "2idJm-i1N6nd"
+ },
+ "source": [
+ "### Time to train\n",
+ "\n",
+ "As the name of this section suggests, the time has come to bring all the previous elements together and start training with ``Trainer``."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 844
},
+ "id": "FT-PSxj7LLKg",
+ "outputId": "09f58e6b-2aaa-48c0-be61-4a4f7e20a1dc"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 6,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "L9_dS8-SvAPP",
- "outputId": "2187d913-c167-4089-f4d5-933b0f07e71a"
- },
- "outputs": [
- {
- "data": {
- "text/plain": [
- "Dataset({\n",
- " features: ['tokens', 'ner_tags'],\n",
- " num_rows: 119\n",
- "})"
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "You're using a DistilBertTokenizerFast tokenizer. Please note that with a fast tokenizer, using the `__call__` method is faster than using a method to encode the text followed by a call to the `pad` method to get a padded encoding.\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "
"
],
- "source": [
- "dataset"
+ "text/plain": [
+ ""
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "markdown",
- "metadata": {
- "id": "ITgB-mzZz3vX"
- },
- "source": [
- "Next, we will use the following code taking advantage of the ``DatasetDict`` option ``Features`` to convert it to the format required by Argilla in order to log it.\n",
- "\n",
- "The three elements that our data must have for Token Classifications are the following:\n",
- "\n",
- "* **text**: the complete string.\n",
- "* **tokens**: the sequence of tokens.\n",
- "* **annotation**: a tuple formed by the tag, the start position and the end position.\n",
- "\n",
- "> ⚠️ **Be careful:** Each execution will upload and add your annotations again without being overwritten."
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "/usr/local/lib/python3.10/dist-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.\n",
+ " _warn_prf(average, modifier, msg_start, len(result))\n",
+ "/usr/local/lib/python3.10/dist-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 due to no predicted samples. Use `zero_division` parameter to control this behavior.\n",
+ " _warn_prf(average, modifier, msg_start, len(result))\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": 79,
- "metadata": {
- "id": "K7WFV0vaHhyd"
- },
- "outputs": [],
- "source": [
- "# Create a function to read the sequences\n",
- "def parse_entities(record):\n",
- " current_entity = None # to check if current entity in process\n",
- " current_info = [] # to save the information used in the tuple for the whole sentence\n",
- " char_position = 0\n",
- " entities = [] # final list to save the tuples\n",
- "\n",
- " # Iterate over the tokens and ner tags\n",
- " for i in range(len(record[\"ner_tags\"])):\n",
- " token = record[\"tokens\"][i]\n",
- " ner_tag = dataset.features[\"ner_tags\"].feature.names[record[\"ner_tags\"][i]]\n",
- "\n",
- " if ner_tag.startswith(\"B-\"):\n",
- " if current_entity:\n",
- " current_info.append(current_entity)\n",
- " current_entity = {\"word\": token, \"start\": char_position, \"tag\": ner_tag[2:]}\n",
- " char_position += len(token) + 1\n",
- "\n",
- " elif ner_tag.startswith(\"I-\"):\n",
- " if current_entity:\n",
- " current_entity[\"word\"] += \" \" + token\n",
- " char_position += len(token) + 1\n",
- "\n",
- " elif ner_tag == \"O\":\n",
- " char_position += len(token) + 1\n",
- "\n",
- " # Add the last entity if it exists\n",
- " if current_entity:\n",
- " current_info.append(current_entity)\n",
- "\n",
- " # Calculate the end positions for each entity\n",
- " for entity in current_info:\n",
- " entity[\"end\"] = entity[\"start\"] + len(entity[\"word\"])\n",
- "\n",
- " for entity in current_info:\n",
- " entities.append((entity[\"tag\"], entity[\"start\"], entity[\"end\"]))\n",
- "\n",
- " return entities"
+ "data": {
+ "text/plain": [
+ "TrainOutput(global_step=80, training_loss=0.45428856909275056, metrics={'train_runtime': 14.9864, 'train_samples_per_second': 158.811, 'train_steps_per_second': 5.338, 'total_flos': 32769159790410.0, 'train_loss': 0.45428856909275056, 'epoch': 20.0})"
]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "trainer = Trainer(\n",
+ " model=model,\n",
+ " args=training_args,\n",
+ " train_dataset=tokenized_dataset[\"train\"],\n",
+ " eval_dataset=tokenized_dataset[\"test\"],\n",
+ " tokenizer=tokenizer,\n",
+ " data_collator=data_collator,\n",
+ " compute_metrics=compute_metrics,\n",
+ ")\n",
+ "\n",
+ "# Train.\n",
+ "trainer.train()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "ZCCiyS9llFq6"
+ },
+ "source": [
+ "The `evaluate` method will allow you to evaluate again on the validation set or on another dataset (e.g. if you have train, validation and test)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 193
},
+ "id": "tsX1Z9r_PjT1",
+ "outputId": "daf2665e-5bb7-4948-8267-788b503c07e9"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "WsKBFFAiYv3D"
- },
- "outputs": [],
- "source": [
- "# Write a loop to iterate over each row of your dataset and add the text, tokens, and tuple\n",
- "records = [\n",
- " rg.TokenClassificationRecord(\n",
- " text=\" \".join(row[\"tokens\"]),\n",
- " tokens=row[\"tokens\"],\n",
- " annotation=parse_entities(row),\n",
- " )\n",
- " for row in dataset\n",
- "]\n",
- "\n",
- "# Log the records with the name of your choice\n",
- "rg.log(records, \"spacy_sm_wnut17\")"
+ "data": {
+ "text/html": [
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " [1/1 : < :]\n",
+ "
\n",
+ " "
+ ],
+ "text/plain": [
+ ""
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "markdown",
- "metadata": {
- "id": "3Ydx5se82s_n"
- },
- "source": [
- "So now you will be able to check your annotations in a much more visual way and even edit them if necessary.\n",
- "\n",
- "\n",
- "\n",
- "In addition, **Argilla** also has more options, e.g. to extract [metrics](https://docs.v1.argilla.io/en/latest/reference/python/python_metrics.html) such as the one shown below.\n",
- "\n",
- "\n"
+ "data": {
+ "text/plain": [
+ "{'eval_loss': 1.5393034219741821,\n",
+ " 'eval_precision': 0.2773109243697479,\n",
+ " 'eval_recall': 0.21019108280254778,\n",
+ " 'eval_f1': 0.2391304347826087,\n",
+ " 'eval_accuracy': 0.7450331125827815,\n",
+ " 'eval_runtime': 0.0918,\n",
+ " 'eval_samples_per_second': 326.934,\n",
+ " 'eval_steps_per_second': 10.898,\n",
+ " 'epoch': 20.0}"
]
+ },
+ "execution_count": 14,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "trainer.evaluate()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "0NYZJgRORyjm"
+ },
+ "source": [
+ "> 🔮 **Try to predict**\n",
+ "\n",
+ "> When you have created your model and are happy with it, test it yourself with your own text.\n",
+ "\n",
+ "```python\n",
+ "# Replace this with the directory where it was saved\n",
+ "model_checkpoint = \"your-path\"\n",
+ "token_classifier = pipeline(\"token-classification\", model=model_checkpoint, aggregation_strategy=\"simple\")\n",
+ "token_classifier(\"I heard Madrid is wonderful in spring.\")\n",
+ "\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "4IDnpIN5ZE6u"
+ },
+ "source": [
+ "## 📝✔️ Summary\n",
+ "\n",
+ "In this tutorial, we have learned how to upload our training dataset to Argilla in order to visualise the data it contains and the NER tags it uses and how to fine-tune a BERT model for NER recognition using ``transformers``. This can be very useful to learn the basics of BERT pre-models and, from there, to develop your skills further and try out different ones that may give better results.\n",
+ "\n",
+ "💪Cheers!"
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "provenance": []
+ },
+ "gpuClass": "standard",
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "name": "python"
+ },
+ "widgets": {
+ "application/vnd.jupyter.widget-state+json": {
+ "00b9010d8b794a7594813f6894e5abfd": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 542
- },
- "id": "uIg23A-uRX94",
- "outputId": "0a9fed8e-6ae8-4a96-c40f-420f61f55c73"
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- "\n",
- "\n",
- "\n",
- "
\n",
- "\n",
- "Tip\n",
- "\n",
- "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
- "\n",
- "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
- "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
- "
"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {
- "papermill": {
- "duration": 0.004557,
- "end_time": "2023-11-15T10:59:17.364909",
- "exception": false,
- "start_time": "2023-11-15T10:59:17.360352",
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "First let's install our dependencies and import the necessary libraries:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "bbbd3c38-1e3c-4d7f-97e6-22dd3b668fc8",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": 3.857186,
- "end_time": "2023-11-15T10:59:21.228277",
- "exception": false,
- "start_time": "2023-11-15T10:59:17.371091",
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [],
- "source": [
- "!pip install argilla\n",
- "!pip install datasets"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 1,
- "id": "51b3b93c",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [],
- "source": [
- "import argilla as rg\n",
- "from argilla._constants import DEFAULT_API_KEY\n",
- "from datasets import load_dataset"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "c330bed5-38d1-45bf-b871-98e629ab3af8",
- "metadata": {
- "editable": true,
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "source": [
- "In order to run this notebook we will need some credentials to push and load datasets from `Argilla` and 🤗`hub`, let's set them in the following cell:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 2,
- "id": "ca4fd7ae-7e31-405e-84c1-974828a903bd",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": [
- "parameters"
- ]
- },
- "outputs": [],
- "source": [
- "# Argilla credentials\n",
- "api_url = \"http://localhost:6900\" # \"https://.hf.space\"\n",
- "api_key = DEFAULT_API_KEY # admin.apikey\n",
- "# Huggingface credentials\n",
- "hf_token = \"hf_...\""
- ]
- },
- {
- "cell_type": "markdown",
- "id": "3464037d-e21e-4db4-bfb4-e300fe4c9e53",
- "metadata": {},
- "source": [
- "Log to argilla:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "728e6af7",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [],
- "source": [
- "#papermill_description=logging-to-argilla\n",
- "rg.init(\n",
- " api_url=api_url,\n",
- " api_key=api_key\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Enable Telemetry\n",
- "\n",
- "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../../../reference/telemetry.md) page."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "try:\n",
- " from argilla.utils.telemetry import tutorial_running\n",
- " tutorial_running()\n",
- "except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "f50177aa-4c94-4b8f-8293-a3710e3cb0ca",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "## Configure a `FeedbackDataset` "
- ]
- },
- {
- "cell_type": "markdown",
- "id": "529f4e77-6c05-4d59-b153-170acee36b97",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "For this tutorial we will use the [ag_news](https://huggingface.co/datasets/ag_news) dataset which can be downloaded from the 🤗`hub`. We will load only the first 1000 items from the training sample."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 6,
- "id": "8b01b473-a815-427c-986e-6c2bedf1b5d7",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [
- {
- "data": {
- "text/plain": [
- "Dataset({\n",
- " features: ['text', 'label'],\n",
- " num_rows: 1000\n",
- "})"
- ]
- },
- "execution_count": 6,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "ds = load_dataset(\"ag_news\", split=\"train[:1000]\")\n",
- "ds"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "5b4eb4ff-9530-4542-9975-25d0f4f725c1",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "We will just load the first 1000 records for this tutorial, but feel free to test the full dataset."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "6d2e7997-62c6-49b3-9462-acd6143346a1",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "This dataset contains a collection of news articles (we can see the content in the `text` column), which have been asigned one of the following classification `labels`: *World (0), Sports (1), Business (2), Sci/Tech (3)*.\n",
- "\n",
- "Let's use the [task templates](https://docs.v1.argilla.io/en/latest/practical_guides/create_update_dataset/create_dataset.html#task-templates) to create a feedback dataset ready for `text-classification`."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 20,
- "id": "7294b709",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [
- {
- "data": {
- "text/plain": [
- "FeedbackDataset(\n",
- " fields=[TextField(name='text', title='Text', required=True, type='text', use_markdown=False)]\n",
- " questions=[LabelQuestion(name='label', title='Label', description='Classify the text by selecting the correct label from the given list of labels.', required=True, type='label_selection', labels=['World', 'Sports', 'Business', 'Sci/Tech'], visible_labels=None)]\n",
- " guidelines=Classify the articles into one of the four categories.)\n",
- " metadata_properties=[])\n",
- ")"
- ]
- },
- "execution_count": 20,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "feedback_dataset = rg.FeedbackDataset.for_text_classification(\n",
- " labels=[\"World\", \"Sports\", \"Business\", \"Sci/Tech\"],\n",
- " guidelines=\"Classify the articles into one of the four categories.\",\n",
- ")\n",
- "feedback_dataset"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "5a5238ea",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "We could compare this dataset with the custom configuration we would use previously (we can take a look at the [custom configuration](https://docs.v1.argilla.io/en/latest/practical_guides/create_update_dataset/create_dataset.html#custom-configuration) for more information on the creation of a `FeedbackDataset` when we want a finer control):"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 8,
- "id": "f4823931-39a2-4942-ba83-894d62e0b7cc",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [
- {
- "data": {
- "text/plain": [
- "FeedbackDataset(\n",
- " fields=[TextField(name='text', title='Text from the article', required=True, type='text', use_markdown=False)]\n",
- " questions=[LabelQuestion(name='label', title='In which category does this article fit?', description=None, required=True, type='label_selection', labels={'World': '0', 'Sports': '1', 'Business': '2', 'Sci/Tech': '3'}, visible_labels=None)]\n",
- " guidelines=Classify the articles into one of the four categories.)\n",
- " metadata_properties=[])\n",
- ")"
- ]
- },
- "execution_count": 8,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "feedback_dataset_long = rg.FeedbackDataset(\n",
- " guidelines=\"Classify the articles into one of the four categories.\",\n",
- " fields=[\n",
- " rg.TextField(name=\"text\", title=\"Text from the article\"),\n",
- " ],\n",
- " questions=[\n",
- " rg.LabelQuestion(\n",
- " name=\"label\",\n",
- " title=\"In which category does this article fit?\",\n",
- " labels={\"World\": \"0\", \"Sports\": \"1\", \"Business\": \"2\", \"Sci/Tech\": \"3\"},\n",
- " required=True,\n",
- " visible_labels=None\n",
- " )\n",
- " ]\n",
- ")\n",
- "feedback_dataset_long"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "77ad10bc-d908-41ba-9a5c-c9cc9d56b67f",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "## Add `FeedbackRecords` \n",
- "\n",
- "### From a Hugging Face `dataset` "
- ]
- },
- {
- "cell_type": "markdown",
- "id": "9c194d68-6a87-4bc0-9d4e-f2afdb0561d9",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "The next step once we have our `FeedbackDataset` created is adding the [FeedbackRecords](https://docs.v1.argilla.io/en/latest/getting_started/cheatsheet.html#create-records) to it."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "5110f19b-8bf2-4d48-8a1f-ffd89cfa20f5",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "In order to create our records we can just loop over the items in the `datasets.Dataset`."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 9,
- "id": "825342c7-ef43-4747-8523-18f3e3fc6bd5",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [],
- "source": [
- "records = []\n",
- "for i, item in enumerate(ds):\n",
- " records.append(\n",
- " rg.FeedbackRecord(\n",
- " fields={\n",
- " \"text\": item[\"text\"],\n",
- " },\n",
- " external_id=f\"record-{i}\"\n",
- " )\n",
- " )\n",
- "\n",
- "# We can add an external_id to each record to identify it later."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "3eb1ae7a-01f9-439b-87ac-6d0064164c80",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "### From a `pandas.DataFrame` "
- ]
- },
- {
- "cell_type": "markdown",
- "id": "dba19b65-60d3-4507-aca6-1b4b9bc77d08",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "If we had our data in a different format, let's say a `csv` file, maybe it's more direct to read the data using pandas for that.\n",
- "\n",
- "We will transform our dataset to pandas format for this example, and the remaining `FeedbackRecord` creation remains just the same:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 10,
- "id": "82beb35b-264f-491a-a6be-bd53f40fb509",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- "
\n",
- "\n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "
text
\n",
- "
label
\n",
- "
\n",
- " \n",
- " \n",
- "
\n",
- "
0
\n",
- "
Wall St. Bears Claw Back Into the Black (Reute...
\n",
- "
2
\n",
- "
\n",
- "
\n",
- "
1
\n",
- "
Carlyle Looks Toward Commercial Aerospace (Reu...
\n",
- "
2
\n",
- "
\n",
- "
\n",
- "
2
\n",
- "
Oil and Economy Cloud Stocks' Outlook (Reuters...
\n",
- "
2
\n",
- "
\n",
- "
\n",
- "
3
\n",
- "
Iraq Halts Oil Exports from Main Southern Pipe...
\n",
- "
2
\n",
- "
\n",
- "
\n",
- "
4
\n",
- "
Oil prices soar to all-time record, posing new...
\n",
- "
2
\n",
- "
\n",
- " \n",
- "
\n",
- "
"
- ],
- "text/plain": [
- " text label\n",
- "0 Wall St. Bears Claw Back Into the Black (Reute... 2\n",
- "1 Carlyle Looks Toward Commercial Aerospace (Reu... 2\n",
- "2 Oil and Economy Cloud Stocks' Outlook (Reuters... 2\n",
- "3 Iraq Halts Oil Exports from Main Southern Pipe... 2\n",
- "4 Oil prices soar to all-time record, posing new... 2"
- ]
- },
- "execution_count": 10,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "df_dataset = ds.to_pandas()\n",
- "df_dataset.head()"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "7c7b33af-297c-44ab-981c-b62cf1406843",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "outputs": [],
- "source": [
- "records_pandas = []\n",
- "for i, item in df_dataset.iterrows():\n",
- " records_pandas.append(\n",
- " rg.FeedbackRecord(\n",
- " fields={\n",
- " \"text\": item[\"text\"],\n",
- " },\n",
- " external_id=f\"record-{i}\"\n",
- " )\n",
- " )"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "dd3f1c38-3b56-46e8-8cac-7b3d21beb38d",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "Let's add our records to the dataset:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 21,
- "id": "9bd0ec1c-1fcb-4d4f-a0a2-4f581a670e8b",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [],
- "source": [
- "feedback_dataset.add_records(records)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "16fe9266-7a03-4be6-8789-b141854bcdda",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "By now we have our dataset with the texts ready to be labeled, let's push it to `Argilla`."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "b867eb5b-0b5a-42b1-88f2-f5b67519243e",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "## Save and load a `FeedbackDataset` \n",
- "\n",
- "### From `Argilla` "
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "8f5581dc-8509-484e-ad4e-c73d15227cc9",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [],
- "source": [
- "#papermill_description=push-dataset-to-argilla\n",
- "try:\n",
- " # delete old dataset\n",
- " remote_dataset = feedback_dataset.from_argilla(name=\"end2end_textclassification\", workspace=\"argilla\")\n",
- " remote_dataset.delete()\n",
- "except:\n",
- " pass\n",
- "remote_dataset = feedback_dataset.push_to_argilla(name=\"end2end_textclassification\", workspace=\"argilla\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "aff0f7fd-722f-4f92-912d-029b134848f3",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "If we go to our `Argilla` instance we should see a similar screen like the following."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "827fce93",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- ""
- ]
- },
- {
- "cell_type": "markdown",
- "id": "e5f09e2a-e192-4526-a686-b4d516ca5066",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "Where we can see the *Text from the article* we wanted, and the different labels to choose from."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "fd877169-bd9a-4596-981d-47d2ff4c7c24",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "We can now download the dataset from `Argilla` just to check it:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": 15,
- "id": "fbae2025-c4fe-4e0c-b445-9ee308baa4a5",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [
- {
- "data": {
- "text/plain": [
- "RemoteFeedbackDataset(\n",
- " id=52b0dfc2-ed85-4805-923c-5d51b51ec4c9\n",
- " name=end2end_textclassification\n",
- " workspace=Workspace(id=ce760ed7-0fdf-4d79-b9b7-1c0e4ea896cd, name=argilla, inserted_at=2023-11-23 09:46:05.591993, updated_at=2023-11-23 09:46:05.591993)\n",
- " url=http://localhost:6900/dataset/52b0dfc2-ed85-4805-923c-5d51b51ec4c9/annotation-mode\n",
- " fields=[RemoteTextField(id=UUID('2835bf0e-1259-45b9-a97c-f9b671395563'), client=None, name='text', title='Text', required=True, type='text', use_markdown=False)]\n",
- " questions=[RemoteLabelQuestion(id=UUID('bb6fc4f0-e4b7-480c-84a1-df717de4ac97'), client=None, name='label', title='Label', description=None, required=True, type='label_selection', labels=['World', 'Sports', 'Business', 'Sci/Tech'], visible_labels=None)]\n",
- " guidelines=Classify the articles into one of the four categories.\n",
- " metadata_properties=[]\n",
- ")"
- ]
- },
- "execution_count": 15,
- "metadata": {},
- "output_type": "execute_result"
- }
- ],
- "source": [
- "remote_dataset = rg.FeedbackDataset.from_argilla(\"end2end_textclassification\", workspace=\"argilla\")\n",
- "remote_dataset"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "1f16d868-a9df-4f01-85ac-06bfcee19101",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "### From Hugging Face hub \n",
- "\n",
- "If we wanted to share our dataset with the world, we could use the Huggingface hub for it."
- ]
- },
- {
- "cell_type": "markdown",
- "id": "24aae42a-6b0f-40eb-9d92-4aafcb735844",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "First we need to login to huggingface. The following cell will log us with our previous token.\n",
- "\n",
- "If we don't have one already, we can obtain it from [here](https://huggingface.co/docs/hub/security-tokens) (remember to set the *write* access)."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "1fabcc89-323f-4f4f-9ab9-4b778dd93474",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [],
- "source": [
- "from huggingface_hub import login\n",
- "\n",
- "login(token=hf_token)"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "694c381d-7f7d-4b2d-938a-20df260ec957",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "And now we can just call the method on the `FeedbackDataset`."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "1c471559-916c-4bf8-b115-3d1ba0be621b",
- "metadata": {
- "editable": true,
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [],
- "source": [
- "#papermill_description=push-dataset-to-huggingface\n",
- "remote_dataset.push_to_huggingface(\"argilla/end2end_textclassification\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "761250bc",
- "metadata": {},
- "source": [
- "We can now download the dataset from Hugging Face just to check it:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "id": "c5405ba2",
- "metadata": {
- "editable": true,
- "slideshow": {
- "slide_type": ""
- },
- "tags": []
- },
- "outputs": [],
- "source": [
- "local_dataset = rg.FeedbackDataset.from_huggingface(\"argilla/end2end_textclassification\")"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "408cb3c9-c784-45c8-b206-058a23b801d8",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "## Conclusion"
- ]
- },
- {
- "cell_type": "markdown",
- "id": "6cdb1244-1589-4e85-be36-e0c959817c6d",
- "metadata": {
- "papermill": {
- "duration": null,
- "end_time": null,
- "exception": null,
- "start_time": null,
- "status": "completed"
- },
- "tags": []
- },
- "source": [
- "In this tutorial we created an `Argilla` `FeedbackDataset` for text classification, starting from [ag_news](https://huggingface.co/datasets/ag_news).\n",
- "\n",
- "We created a `FeedbackDataset` for text classification with a `LabelQuestion`, from data stored as a `datasets.Dataset` and a `pandas.DataFrame`.\n",
- "This dataset was pushed both to `Argilla` where we can curate and label the records, and finally pushed it to the 🤗`hub`.\n",
- "\n",
- "To learn more about how to work with the `FeedbackDataset` check the [cheatsheet](https://docs.v1.argilla.io/en/latest/getting_started/cheatsheet.html#cheatsheet). To continue with assigning records to annotators, you can refer to the [next tutorial](./assign-records-002.ipynb)."
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": "Python 3 (ipykernel)",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.10.13"
- },
- "papermill": {
- "default_parameters": {},
- "duration": 5.95317,
- "end_time": "2023-11-15T10:59:21.949504",
- "environment_variables": {},
- "exception": null,
- "input_path": "/home/agustin/github_repos/argilla-io/argilla/docs/_source/practical_guides/examples/text_classification/text-classification-create-dataset.ipynb",
- "output_path": "/home/agustin/github_repos/argilla-io/argilla/docs/_source/practical_guides/examples/text_classification/text-classification-create-dataset.ipynb",
- "parameters": {},
- "start_time": "2023-11-15T10:59:15.996334",
- "version": "2.5.0"
- }
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "id": "fc0c600b",
+ "metadata": {
+ "papermill": {
+ "duration": 0.046655,
+ "end_time": "2023-11-15T10:59:17.308341",
+ "exception": false,
+ "start_time": "2023-11-15T10:59:17.261686",
+ "status": "completed"
},
- "nbformat": 4,
- "nbformat_minor": 5
+ "tags": []
+ },
+ "source": [
+ "# Creating a `FeedbackDataset`\n",
+ "\n",
+ "This tutorial is part of a series in which we will get to know the `FeedbackDataset`. Before starting this tutorial, you need to do the tutorial on [configuring users and workspaces](./configure-users-and-workspaces-000.ipynb). In this step, we will show how to configure a `FeedbackDataset` and add `FeedbackRecords` to it. If you need additional context, consult [our practical guide on creating a dataset](../../../../practical_guides/create_update_dataset/create_dataset.md).\n",
+ "\n",
+ "\n",
+ "\n",
+ "We will start by creating a basic dataset using the [ag_news](https://huggingface.co/datasets/ag_news) dataset as an example and push it to `Argilla` and the Hugging Face `hub`."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e557a44e-26fb-4b52-91ec-2b49d550a5cd",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": 0.011949,
+ "end_time": "2023-11-15T10:59:17.340523",
+ "exception": false,
+ "start_time": "2023-11-15T10:59:17.328574",
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "source": [
+ "## Table of Contents\n",
+ "\n",
+ "1. [Configure a FeedbackDataset](#Configure-a-FeedbackDataset)\n",
+ "2. [Add FeedbackRecords](#Add-FeedbackRecords)\n",
+ " 1. [From a Hugging Face dataset](#From-a-Hugging-Face-dataset)\n",
+ " 2. [From a pandas.DataFrame](#From-a-pandas.DataFrame)\n",
+ "3. [Save and load a FeedbackDataset](#Save-and-load-a-FeedbackDataset)\n",
+ " 1. [Push our FeedbackDataset to Argilla](#From-Argilla)\n",
+ " 2. [Push our FeedbackDataset to the Hugging Face hub](#From-Hugging-Face-hub)\n",
+ "4. [Conclusion](#Conclusion)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "566b7b82",
+ "metadata": {
+ "papermill": {
+ "duration": 0.007744,
+ "end_time": "2023-11-15T10:59:17.355040",
+ "exception": false,
+ "start_time": "2023-11-15T10:59:17.347296",
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "## Running Argilla\n",
+ "\n",
+ "For this tutorial, you will need to have an Argilla server running. There are two main options for deploying and running Argilla:\n",
+ "\n",
+ "**Deploy Argilla on Hugging Face Spaces:** If you want to run tutorials with external notebooks (e.g., Google Colab) and you have an account on Hugging Face, you can deploy Argilla on Spaces with a few clicks:\n",
+ "\n",
+ "[](https://huggingface.co/new-space?template=argilla/argilla-template-space)\n",
+ "\n",
+ "For details about configuring your deployment, check the [official Hugging Face Hub guide](https://huggingface.co/docs/hub/spaces-sdks-docker-argilla).\n",
+ "\n",
+ "**Launch Argilla using Argilla's quickstart Docker image**: This is the recommended option if you want [Argilla running on your local machine](../../../../getting_started/quickstart.md). Note that this option will only let you run the tutorial locally and not with an external notebook service.\n",
+ "\n",
+ "For more information on deployment options, please check the Deployment section of the documentation.\n",
+ "\n",
+ "
\n",
+ "\n",
+ "Tip\n",
+ "\n",
+ "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
+ "\n",
+ "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
+ "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "7fb27b941602401d91542211134fc71a",
+ "metadata": {
+ "papermill": {
+ "duration": 0.004557,
+ "end_time": "2023-11-15T10:59:17.364909",
+ "exception": false,
+ "start_time": "2023-11-15T10:59:17.360352",
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "First let's install our dependencies and import the necessary libraries:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "bbbd3c38-1e3c-4d7f-97e6-22dd3b668fc8",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": 3.857186,
+ "end_time": "2023-11-15T10:59:21.228277",
+ "exception": false,
+ "start_time": "2023-11-15T10:59:17.371091",
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "!pip install argilla\n",
+ "!pip install datasets"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "id": "51b3b93c",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "import argilla as rg\n",
+ "from argilla._constants import DEFAULT_API_KEY\n",
+ "from datasets import load_dataset"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "c330bed5-38d1-45bf-b871-98e629ab3af8",
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "source": [
+ "In order to run this notebook we will need some credentials to push and load datasets from `Argilla` and 🤗`hub`, let's set them in the following cell:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 2,
+ "id": "ca4fd7ae-7e31-405e-84c1-974828a903bd",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": [
+ "parameters"
+ ]
+ },
+ "outputs": [],
+ "source": [
+ "# Argilla credentials\n",
+ "api_url = \"http://localhost:6900\" # \"https://.hf.space\"\n",
+ "api_key = DEFAULT_API_KEY # admin.apikey\n",
+ "# Huggingface credentials\n",
+ "hf_token = \"hf_...\""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3464037d-e21e-4db4-bfb4-e300fe4c9e53",
+ "metadata": {},
+ "source": [
+ "Log to argilla:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "728e6af7",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# papermill_description=logging-to-argilla\n",
+ "rg.init(api_url=api_url, api_key=api_key)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "acae54e37e7d407bbb7b55eff062a284",
+ "metadata": {},
+ "source": [
+ "### Enable Telemetry\n",
+ "\n",
+ "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../../../reference/telemetry.md) page."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "9a63283cbaf04dbcab1f6479b197f3a8",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
+ " tutorial_running()\n",
+ "except ImportError:\n",
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "f50177aa-4c94-4b8f-8293-a3710e3cb0ca",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "## Configure a `FeedbackDataset` "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "529f4e77-6c05-4d59-b153-170acee36b97",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "For this tutorial we will use the [ag_news](https://huggingface.co/datasets/ag_news) dataset which can be downloaded from the 🤗`hub`. We will load only the first 1000 items from the training sample."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 6,
+ "id": "8b01b473-a815-427c-986e-6c2bedf1b5d7",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "Dataset({\n",
+ " features: ['text', 'label'],\n",
+ " num_rows: 1000\n",
+ "})"
+ ]
+ },
+ "execution_count": 6,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "ds = load_dataset(\"ag_news\", split=\"train[:1000]\")\n",
+ "ds"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5b4eb4ff-9530-4542-9975-25d0f4f725c1",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "We will just load the first 1000 records for this tutorial, but feel free to test the full dataset."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6d2e7997-62c6-49b3-9462-acd6143346a1",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "This dataset contains a collection of news articles (we can see the content in the `text` column), which have been asigned one of the following classification `labels`: *World (0), Sports (1), Business (2), Sci/Tech (3)*.\n",
+ "\n",
+ "Let's use the [task templates](https://docs.v1.argilla.io/en/latest/practical_guides/create_update_dataset/create_dataset.html#task-templates) to create a feedback dataset ready for `text-classification`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 20,
+ "id": "7294b709",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "FeedbackDataset(\n",
+ " fields=[TextField(name='text', title='Text', required=True, type='text', use_markdown=False)]\n",
+ " questions=[LabelQuestion(name='label', title='Label', description='Classify the text by selecting the correct label from the given list of labels.', required=True, type='label_selection', labels=['World', 'Sports', 'Business', 'Sci/Tech'], visible_labels=None)]\n",
+ " guidelines=Classify the articles into one of the four categories.)\n",
+ " metadata_properties=[])\n",
+ ")"
+ ]
+ },
+ "execution_count": 20,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "feedback_dataset = rg.FeedbackDataset.for_text_classification(\n",
+ " labels=[\"World\", \"Sports\", \"Business\", \"Sci/Tech\"],\n",
+ " guidelines=\"Classify the articles into one of the four categories.\",\n",
+ ")\n",
+ "feedback_dataset"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5a5238ea",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "We could compare this dataset with the custom configuration we would use previously (we can take a look at the [custom configuration](https://docs.v1.argilla.io/en/latest/practical_guides/create_update_dataset/create_dataset.html#custom-configuration) for more information on the creation of a `FeedbackDataset` when we want a finer control):"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "id": "f4823931-39a2-4942-ba83-894d62e0b7cc",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "FeedbackDataset(\n",
+ " fields=[TextField(name='text', title='Text from the article', required=True, type='text', use_markdown=False)]\n",
+ " questions=[LabelQuestion(name='label', title='In which category does this article fit?', description=None, required=True, type='label_selection', labels={'World': '0', 'Sports': '1', 'Business': '2', 'Sci/Tech': '3'}, visible_labels=None)]\n",
+ " guidelines=Classify the articles into one of the four categories.)\n",
+ " metadata_properties=[])\n",
+ ")"
+ ]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "feedback_dataset_long = rg.FeedbackDataset(\n",
+ " guidelines=\"Classify the articles into one of the four categories.\",\n",
+ " fields=[\n",
+ " rg.TextField(name=\"text\", title=\"Text from the article\"),\n",
+ " ],\n",
+ " questions=[\n",
+ " rg.LabelQuestion(\n",
+ " name=\"label\",\n",
+ " title=\"In which category does this article fit?\",\n",
+ " labels={\"World\": \"0\", \"Sports\": \"1\", \"Business\": \"2\", \"Sci/Tech\": \"3\"},\n",
+ " required=True,\n",
+ " visible_labels=None,\n",
+ " )\n",
+ " ],\n",
+ ")\n",
+ "feedback_dataset_long"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "77ad10bc-d908-41ba-9a5c-c9cc9d56b67f",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "## Add `FeedbackRecords` \n",
+ "\n",
+ "### From a Hugging Face `dataset` "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "9c194d68-6a87-4bc0-9d4e-f2afdb0561d9",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "The next step once we have our `FeedbackDataset` created is adding the [FeedbackRecords](https://docs.v1.argilla.io/en/latest/getting_started/cheatsheet.html#create-records) to it."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "5110f19b-8bf2-4d48-8a1f-ffd89cfa20f5",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "In order to create our records we can just loop over the items in the `datasets.Dataset`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "id": "825342c7-ef43-4747-8523-18f3e3fc6bd5",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "records = []\n",
+ "for i, item in enumerate(ds):\n",
+ " records.append(\n",
+ " rg.FeedbackRecord(\n",
+ " fields={\n",
+ " \"text\": item[\"text\"],\n",
+ " },\n",
+ " external_id=f\"record-{i}\",\n",
+ " )\n",
+ " )\n",
+ "\n",
+ "# We can add an external_id to each record to identify it later."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "3eb1ae7a-01f9-439b-87ac-6d0064164c80",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "### From a `pandas.DataFrame` "
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dba19b65-60d3-4507-aca6-1b4b9bc77d08",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "If we had our data in a different format, let's say a `csv` file, maybe it's more direct to read the data using pandas for that.\n",
+ "\n",
+ "We will transform our dataset to pandas format for this example, and the remaining `FeedbackRecord` creation remains just the same:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "id": "82beb35b-264f-491a-a6be-bd53f40fb509",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
text
\n",
+ "
label
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
0
\n",
+ "
Wall St. Bears Claw Back Into the Black (Reute...
\n",
+ "
2
\n",
+ "
\n",
+ "
\n",
+ "
1
\n",
+ "
Carlyle Looks Toward Commercial Aerospace (Reu...
\n",
+ "
2
\n",
+ "
\n",
+ "
\n",
+ "
2
\n",
+ "
Oil and Economy Cloud Stocks' Outlook (Reuters...
\n",
+ "
2
\n",
+ "
\n",
+ "
\n",
+ "
3
\n",
+ "
Iraq Halts Oil Exports from Main Southern Pipe...
\n",
+ "
2
\n",
+ "
\n",
+ "
\n",
+ "
4
\n",
+ "
Oil prices soar to all-time record, posing new...
\n",
+ "
2
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " text label\n",
+ "0 Wall St. Bears Claw Back Into the Black (Reute... 2\n",
+ "1 Carlyle Looks Toward Commercial Aerospace (Reu... 2\n",
+ "2 Oil and Economy Cloud Stocks' Outlook (Reuters... 2\n",
+ "3 Iraq Halts Oil Exports from Main Southern Pipe... 2\n",
+ "4 Oil prices soar to all-time record, posing new... 2"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df_dataset = ds.to_pandas()\n",
+ "df_dataset.head()"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "7c7b33af-297c-44ab-981c-b62cf1406843",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "records_pandas = []\n",
+ "for i, item in df_dataset.iterrows():\n",
+ " records_pandas.append(\n",
+ " rg.FeedbackRecord(\n",
+ " fields={\n",
+ " \"text\": item[\"text\"],\n",
+ " },\n",
+ " external_id=f\"record-{i}\",\n",
+ " )\n",
+ " )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "dd3f1c38-3b56-46e8-8cac-7b3d21beb38d",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "Let's add our records to the dataset:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 21,
+ "id": "9bd0ec1c-1fcb-4d4f-a0a2-4f581a670e8b",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "feedback_dataset.add_records(records)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "16fe9266-7a03-4be6-8789-b141854bcdda",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "By now we have our dataset with the texts ready to be labeled, let's push it to `Argilla`."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "b867eb5b-0b5a-42b1-88f2-f5b67519243e",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "## Save and load a `FeedbackDataset` \n",
+ "\n",
+ "### From `Argilla` "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "8f5581dc-8509-484e-ad4e-c73d15227cc9",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# papermill_description=push-dataset-to-argilla\n",
+ "try:\n",
+ " # delete old dataset\n",
+ " remote_dataset = feedback_dataset.from_argilla(\n",
+ " name=\"end2end_textclassification\", workspace=\"argilla\"\n",
+ " )\n",
+ " remote_dataset.delete()\n",
+ "except:\n",
+ " pass\n",
+ "remote_dataset = feedback_dataset.push_to_argilla(\n",
+ " name=\"end2end_textclassification\", workspace=\"argilla\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "aff0f7fd-722f-4f92-912d-029b134848f3",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "If we go to our `Argilla` instance we should see a similar screen like the following."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "827fce93",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ ""
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "e5f09e2a-e192-4526-a686-b4d516ca5066",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "Where we can see the *Text from the article* we wanted, and the different labels to choose from."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "fd877169-bd9a-4596-981d-47d2ff4c7c24",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "We can now download the dataset from `Argilla` just to check it:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "id": "fbae2025-c4fe-4e0c-b445-9ee308baa4a5",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "RemoteFeedbackDataset(\n",
+ " id=52b0dfc2-ed85-4805-923c-5d51b51ec4c9\n",
+ " name=end2end_textclassification\n",
+ " workspace=Workspace(id=ce760ed7-0fdf-4d79-b9b7-1c0e4ea896cd, name=argilla, inserted_at=2023-11-23 09:46:05.591993, updated_at=2023-11-23 09:46:05.591993)\n",
+ " url=http://localhost:6900/dataset/52b0dfc2-ed85-4805-923c-5d51b51ec4c9/annotation-mode\n",
+ " fields=[RemoteTextField(id=UUID('2835bf0e-1259-45b9-a97c-f9b671395563'), client=None, name='text', title='Text', required=True, type='text', use_markdown=False)]\n",
+ " questions=[RemoteLabelQuestion(id=UUID('bb6fc4f0-e4b7-480c-84a1-df717de4ac97'), client=None, name='label', title='Label', description=None, required=True, type='label_selection', labels=['World', 'Sports', 'Business', 'Sci/Tech'], visible_labels=None)]\n",
+ " guidelines=Classify the articles into one of the four categories.\n",
+ " metadata_properties=[]\n",
+ ")"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "remote_dataset = rg.FeedbackDataset.from_argilla(\n",
+ " \"end2end_textclassification\", workspace=\"argilla\"\n",
+ ")\n",
+ "remote_dataset"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "1f16d868-a9df-4f01-85ac-06bfcee19101",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "### From Hugging Face hub \n",
+ "\n",
+ "If we wanted to share our dataset with the world, we could use the Huggingface hub for it."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "24aae42a-6b0f-40eb-9d92-4aafcb735844",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "First we need to login to huggingface. The following cell will log us with our previous token.\n",
+ "\n",
+ "If we don't have one already, we can obtain it from [here](https://huggingface.co/docs/hub/security-tokens) (remember to set the *write* access)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1fabcc89-323f-4f4f-9ab9-4b778dd93474",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "from huggingface_hub import login\n",
+ "\n",
+ "login(token=hf_token)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "694c381d-7f7d-4b2d-938a-20df260ec957",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "And now we can just call the method on the `FeedbackDataset`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "1c471559-916c-4bf8-b115-3d1ba0be621b",
+ "metadata": {
+ "editable": true,
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "# papermill_description=push-dataset-to-huggingface\n",
+ "remote_dataset.push_to_huggingface(\"argilla/end2end_textclassification\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "761250bc",
+ "metadata": {},
+ "source": [
+ "We can now download the dataset from Hugging Face just to check it:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "c5405ba2",
+ "metadata": {
+ "editable": true,
+ "slideshow": {
+ "slide_type": ""
+ },
+ "tags": []
+ },
+ "outputs": [],
+ "source": [
+ "local_dataset = rg.FeedbackDataset.from_huggingface(\n",
+ " \"argilla/end2end_textclassification\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "408cb3c9-c784-45c8-b206-058a23b801d8",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "## Conclusion"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "6cdb1244-1589-4e85-be36-e0c959817c6d",
+ "metadata": {
+ "papermill": {
+ "duration": null,
+ "end_time": null,
+ "exception": null,
+ "start_time": null,
+ "status": "completed"
+ },
+ "tags": []
+ },
+ "source": [
+ "In this tutorial we created an `Argilla` `FeedbackDataset` for text classification, starting from [ag_news](https://huggingface.co/datasets/ag_news).\n",
+ "\n",
+ "We created a `FeedbackDataset` for text classification with a `LabelQuestion`, from data stored as a `datasets.Dataset` and a `pandas.DataFrame`.\n",
+ "This dataset was pushed both to `Argilla` where we can curate and label the records, and finally pushed it to the 🤗`hub`.\n",
+ "\n",
+ "To learn more about how to work with the `FeedbackDataset` check the [cheatsheet](https://docs.v1.argilla.io/en/latest/getting_started/cheatsheet.html#cheatsheet). To continue with assigning records to annotators, you can refer to the [next tutorial](./assign-records-002.ipynb)."
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.13"
+ },
+ "papermill": {
+ "default_parameters": {},
+ "duration": 5.95317,
+ "end_time": "2023-11-15T10:59:21.949504",
+ "environment_variables": {},
+ "exception": null,
+ "input_path": "/home/agustin/github_repos/argilla-io/argilla/docs/_source/practical_guides/examples/text_classification/text-classification-create-dataset.ipynb",
+ "output_path": "/home/agustin/github_repos/argilla-io/argilla/docs/_source/practical_guides/examples/text_classification/text-classification-create-dataset.ipynb",
+ "parameters": {},
+ "start_time": "2023-11-15T10:59:15.996334",
+ "version": "2.5.0"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
}
diff --git a/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/filter-and-query-008.ipynb b/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/filter-and-query-008.ipynb
index a839262bf7..9cca459466 100644
--- a/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/filter-and-query-008.ipynb
+++ b/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/filter-and-query-008.ipynb
@@ -148,9 +148,12 @@
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -172,9 +175,15 @@
"metadata": {},
"outputs": [],
"source": [
- "dataset_remote_with_metadata = rg.FeedbackDataset.from_argilla(\"end2end_textclassification_with_metadata\")\n",
- "dataset_remote_with_vectors = rg.FeedbackDataset.from_argilla(\"end2end_textclassification_with_vectors\")\n",
- "dataset_remote_with_suggestions_and_responses = rg.FeedbackDataset.from_argilla(\"end2end_textclassification_with_suggestions_and_responses\")"
+ "dataset_remote_with_metadata = rg.FeedbackDataset.from_argilla(\n",
+ " \"end2end_textclassification_with_metadata\"\n",
+ ")\n",
+ "dataset_remote_with_vectors = rg.FeedbackDataset.from_argilla(\n",
+ " \"end2end_textclassification_with_vectors\"\n",
+ ")\n",
+ "dataset_remote_with_suggestions_and_responses = rg.FeedbackDataset.from_argilla(\n",
+ " \"end2end_textclassification_with_suggestions_and_responses\"\n",
+ ")"
]
},
{
@@ -296,19 +305,9 @@
"source": [
"filtered_records = dataset_remote_with_metadata.filter_by(\n",
" metadata_filters=[\n",
- " rg.TermsMetadataFilter(\n",
- " name=\"group\",\n",
- " values=[\"group-1\", \"group-2\"]\n",
- " ),\n",
- " rg.IntegerMetadataFilter(\n",
- " name=\"length\",\n",
- " le=282\n",
- " ),\n",
- " rg.FloatMetadataFilter(\n",
- " name=\"length_std\",\n",
- " ge=204, \n",
- " le=290\n",
- " ), \n",
+ " rg.TermsMetadataFilter(name=\"group\", values=[\"group-1\", \"group-2\"]),\n",
+ " rg.IntegerMetadataFilter(name=\"length\", le=282),\n",
+ " rg.FloatMetadataFilter(name=\"length_std\", ge=204, le=290),\n",
" ]\n",
")\n",
"\n",
@@ -374,7 +373,7 @@
],
"source": [
"filtered_dataset = dataset_remote_with_metadata.filter_by(response_status=[\"submitted\"])\n",
- "print('Submitted records:', len(filtered_dataset))"
+ "print(\"Submitted records:\", len(filtered_dataset))"
]
},
{
@@ -398,8 +397,10 @@
}
],
"source": [
- "filtered_dataset = dataset_remote_with_metadata.filter_by(response_status=[\"missing\", \"draft\"])\n",
- "print('Pending records:', len(filtered_dataset))"
+ "filtered_dataset = dataset_remote_with_metadata.filter_by(\n",
+ " response_status=[\"missing\", \"draft\"]\n",
+ ")\n",
+ "print(\"Pending records:\", len(filtered_dataset))"
]
},
{
@@ -425,24 +426,14 @@
"source": [
"filtered_dataset = dataset_remote_with_metadata.filter_by(\n",
" metadata_filters=[\n",
- " rg.TermsMetadataFilter(\n",
- " name=\"group\",\n",
- " values=[\"group-1\", \"group-2\"]\n",
- " ),\n",
- " rg.IntegerMetadataFilter(\n",
- " name=\"length\",\n",
- " le=282\n",
- " ),\n",
- " rg.FloatMetadataFilter(\n",
- " name=\"length_std\",\n",
- " ge=204, \n",
- " le=290\n",
- " ),\n",
+ " rg.TermsMetadataFilter(name=\"group\", values=[\"group-1\", \"group-2\"]),\n",
+ " rg.IntegerMetadataFilter(name=\"length\", le=282),\n",
+ " rg.FloatMetadataFilter(name=\"length_std\", ge=204, le=290),\n",
" ],\n",
- " response_status=[\"discarded\"]\n",
+ " response_status=[\"discarded\"],\n",
")\n",
"\n",
- "print('Discarded records:', len(filtered_dataset))"
+ "print(\"Discarded records:\", len(filtered_dataset))"
]
},
{
@@ -491,7 +482,7 @@
"sorted_records = dataset_remote_with_suggestions_and_responses.sort_by(\n",
" [\n",
" SortBy(field=\"updated_at\", order=\"desc\"),\n",
- " SortBy(field=\"metadata.group\", order=\"asc\")\n",
+ " SortBy(field=\"metadata.group\", order=\"asc\"),\n",
" ]\n",
")"
]
@@ -514,7 +505,7 @@
").sort_by(\n",
" [\n",
" SortBy(field=\"updated_at\", order=\"desc\"),\n",
- " SortBy(field=\"metadata.group\", order=\"asc\")\n",
+ " SortBy(field=\"metadata.group\", order=\"asc\"),\n",
" ]\n",
")"
]
@@ -544,7 +535,7 @@
"similar_records = dataset_remote_with_vectors.find_similar_records(\n",
" vector_name=\"sentence_embedding\",\n",
" record=dataset_remote_with_vectors[0],\n",
- " max_results=5\n",
+ " max_results=5,\n",
")"
]
},
@@ -598,9 +589,7 @@
"text_vector = model.encode(text).tolist()\n",
"\n",
"similar_records = dataset_remote_with_vectors.find_similar_records(\n",
- " vector_name=\"sentence_embedding\",\n",
- " value=text_vector,\n",
- " max_results=5\n",
+ " vector_name=\"sentence_embedding\", value=text_vector, max_results=5\n",
")"
]
},
diff --git a/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/train-model-006.ipynb b/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/train-model-006.ipynb
index d8616bc246..a87a92f424 100644
--- a/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/train-model-006.ipynb
+++ b/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/train-model-006.ipynb
@@ -148,9 +148,12 @@
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -172,7 +175,9 @@
"metadata": {},
"outputs": [],
"source": [
- "dataset = rg.FeedbackDataset.from_argilla(\"end2end_textclassification_with_suggestions_and_responses\")"
+ "dataset = rg.FeedbackDataset.from_argilla(\n",
+ " \"end2end_textclassification_with_suggestions_and_responses\"\n",
+ ")"
]
},
{
@@ -190,7 +195,9 @@
"metadata": {},
"outputs": [],
"source": [
- "dataset = rg.FeedbackDataset.from_huggingface(\"argilla/end2end_textclassification_with_suggestions_and_responses\")"
+ "dataset = rg.FeedbackDataset.from_huggingface(\n",
+ " \"argilla/end2end_textclassification_with_suggestions_and_responses\"\n",
+ ")"
]
},
{
@@ -290,9 +297,7 @@
"metadata": {},
"outputs": [],
"source": [
- "trainer.update_config(\n",
- " max_steps=1\n",
- ")"
+ "trainer.update_config(max_steps=1)"
]
},
{
@@ -331,7 +336,7 @@
"def formatting_func(sample):\n",
" text = sample[\"text\"]\n",
" label = sample[\"label\"][0][\"value\"]\n",
- " return(text, label)"
+ " return (text, label)"
]
},
{
@@ -384,9 +389,7 @@
"metadata": {},
"outputs": [],
"source": [
- "trainer.update_config(\n",
- " max_steps=1\n",
- ")"
+ "trainer.update_config(max_steps=1)"
]
},
{
diff --git a/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/use-metrics-007.ipynb b/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/use-metrics-007.ipynb
index 0914cbf6fd..80c5f35d8a 100644
--- a/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/use-metrics-007.ipynb
+++ b/docs/_source/tutorials_and_integrations/tutorials/feedback/end2end_examples/use-metrics-007.ipynb
@@ -149,9 +149,12 @@
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry module is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry module is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -179,7 +182,9 @@
"metadata": {},
"outputs": [],
"source": [
- "dataset = rg.FeedbackDataset.from_huggingface(\"argilla/go_emotions_raw\", split=\"train[:1000]\")"
+ "dataset = rg.FeedbackDataset.from_huggingface(\n",
+ " \"argilla/go_emotions_raw\", split=\"train[:1000]\"\n",
+ ")"
]
},
{
@@ -218,7 +223,13 @@
],
"source": [
"print(\"text:\", dataset[5].fields[\"text\"])\n",
- "print(\"responses:\", [dataset[5].responses[i].values[\"label\"].value for i in range(len(dataset[5].responses))])"
+ "print(\n",
+ " \"responses:\",\n",
+ " [\n",
+ " dataset[5].responses[i].values[\"label\"].value\n",
+ " for i in range(len(dataset[5].responses))\n",
+ " ],\n",
+ ")"
]
},
{
@@ -389,7 +400,9 @@
"metadata": {},
"outputs": [],
"source": [
- "model_metrics = dataset.compute_model_metrics(question_name=\"label\", metric_names=metric.allowed_metrics)"
+ "model_metrics = dataset.compute_model_metrics(\n",
+ " question_name=\"label\", metric_names=metric.allowed_metrics\n",
+ ")"
]
},
{
@@ -445,7 +458,11 @@
"metadata": {},
"outputs": [],
"source": [
- "model_metrics_unified = dataset.compute_model_metrics(question_name=\"label\", metric_names=[\"accuracy\", \"precision\", \"recall\", \"f1-score\"], strategy=\"majority\")"
+ "model_metrics_unified = dataset.compute_model_metrics(\n",
+ " question_name=\"label\",\n",
+ " metric_names=[\"accuracy\", \"precision\", \"recall\", \"f1-score\"],\n",
+ " strategy=\"majority\",\n",
+ ")"
]
},
{
diff --git a/docs/_source/tutorials_and_integrations/tutorials/feedback/fine-tuning-openai-rag-feedback.ipynb b/docs/_source/tutorials_and_integrations/tutorials/feedback/fine-tuning-openai-rag-feedback.ipynb
index e29ccb29df..6196d6e146 100644
--- a/docs/_source/tutorials_and_integrations/tutorials/feedback/fine-tuning-openai-rag-feedback.ipynb
+++ b/docs/_source/tutorials_and_integrations/tutorials/feedback/fine-tuning-openai-rag-feedback.ipynb
@@ -1,797 +1,837 @@
{
- "cells": [
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "FOyRtBFaD8Ti"
- },
- "source": [
- "# 🪄 Fine-tuning and evaluating GPT-3.5 with human feedback for RAG \n",
- "\n",
- "This guide explains how to fine-tune OpenAI's GPT3.5-turbo with your own data and Argilla to improve a RAG (Retrieval Augmented Generation) system. \n",
- "\n",
- "It includes the following steps:\n",
- "\n",
- "- Setting up a RAG pipeline using [LlamaIndex](https://github.com/jerryjliu/llama_index) and [Unstructured](https://github.com/Unstructured-IO/unstructured) to answer questions using a document about Argilla Cloud.\n",
- "- Generating potential questions with LlamaIndex to build a training and test set.\n",
- "- Building a dataset for collecting human written responses with Argilla.\n",
- "- Fine-tuning GPT3.5-turbo with high-quality data.\n",
- "- Evaluating the fine-tuned model vs. the base model with human preference data from Argilla.\n",
- "\n",
- "The goal of the tutorial is to demonstrate how to incorporate human feedback into your LLM development for two critical stages: \n",
- "\n",
- "1. Gathering **high-quality data for fine-tuning**, \n",
- "2. Gathering **human feedback for evaluation of LLM applications**.\n",
- "\n",
- "\n",
- "Given the ongoing debate between Retrieval Augmented Generation (RAG) and fine-tuning, we selected a real-world RAG use case to demonstrate how fine-tuning enhances the style, utility, and relevance of responses within a RAG application. The resulting system will be a Hybrid RAG system (RAG using fine-tuned models) as [described in this article](https://towardsdatascience.com/rag-vs-finetuning-which-is-the-best-tool-to-boost-your-llm-application-94654b1eaba7). \n",
- "\n",
- "The screenshot below displays the evaluation dataset, termed the \"human preference dataset.\" In it, `response-a` is produced by the fine-tuned model, while `response-b` comes from the base GPT-3.5 model. With just minor fine-tuning and without altering the system message, we've directed the LLM's behavior towards generating responses that are more helpful, faithful, friendly, and aligned with our brand.\n",
- "\n",
- "Fine-tuning effectively mitigates common RAG challenges, like the LLM referring to the context using phrases such as \"The context does not provide information about this.\" This enhancement is notable even when we had incorporated directives in the system message to deter such references, like \"2. Avoid phrases such as 'Based on the context, ...' or 'The context information ...'.\" (see Llama Index default prompt later).\n",
- "\n",
- "You can also browse the [datasets hosted with Argilla Hugging Face Spaces](https://huggingface.co/spaces/argilla/fine-tune-chat-gpt). User and password: argilla / 12345678. The dataset for this stage is `customer-assistant` and for the evaluation step is `finetuned-vs-base-preference`.\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "By the end of the tutorial, you'll be using a fine-tuned model for RAG and have a human evaluation workflow in place to continuously evaluate your LLM application (see below for a comparison of the base gpt3.5 vs. the fine-tuned gpt3.5 for this application).\n",
- "\n",
- "\n",
- "\n",
- "\n",
- "Let's get started!\n",
- "\n",
- "## Setup\n",
- "\n",
- "To run this tutorial, you need to [install and launch Argilla](https://docs.v1.argilla.io/en/latest/getting_started/quickstart_installation.html), as well as some other packages."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "j76dPBrN3iR_"
- },
- "outputs": [],
- "source": [
- "%pip install argilla openai datasets llama-index unstructured -qqq"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "RI_lnoziIkww"
- },
- "outputs": [],
- "source": [
- "# Import the needed libraries\n",
- "import os\n",
- "import random\n",
- "from tqdm import tqdm\n",
- "import matplotlib.pyplot as plt\n",
- "\n",
- "import openai\n",
- "\n",
- "import argilla as rg\n",
- "from argilla.feedback import TrainingTask\n",
- "from argilla.feedback import ArgillaTrainer\n",
- "\n",
- "from typing import Union, Tuple, List\n",
- "\n",
- "from llama_index.core import ServiceContext, VectorStoreIndex, download_loader\n",
- "from llama_index.llms.openai import OpenAI\n",
- "from llama_index.core.evaluation import DatasetGenerator\n",
- "\n",
- "from datasets import load_dataset"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the URL and API_KEY:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "kIrT-iWx3kbX"
- },
- "outputs": [],
- "source": [
- "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
- "# Replace api_key if you configured a custom API key\n",
- "# Replace workspace with the name of your workspace\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\",\n",
- " api_key=\"owner.apikey\",\n",
- " workspace=\"admin\"\n",
- ")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# # Set the HF_TOKEN environment variable\n",
- "# import os\n",
- "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
- "\n",
- "# # Replace api_url with the url to your HF Spaces URL\n",
- "# # Replace api_key if you configured a custom API key\n",
- "# # Replace workspace with the name of your workspace\n",
- "# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
- "# api_key=\"owner.apikey\",\n",
- "# workspace=\"admin\",\n",
- "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
- "# )"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "T45WtJ9t9Vzq"
- },
- "outputs": [],
- "source": [
- "# Your openAI key is needed for generation and fine-tuning\n",
- "os.environ['OPENAI_API_KEY'] = 'sk-...'\n",
- "openai.api_key = os.environ[\"OPENAI_API_KEY\"]"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Enable Telemetry\n",
- "\n",
- "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../../reference/telemetry.md) page.\n",
- "\n",
- "```python\n",
- "from argilla.utils.telemetry import tutorial_running\n",
- "\n",
- "tutorial_running()\n",
- "```"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "K4ExAXiKSCu5"
- },
- "source": [
- "## Generating responses with LlamaIndex and GPT3.5\n",
- "\n",
- "We generate responses for the generated questions using [this dataset about Argilla Cloud](https://huggingface.co/datasets/argilla/cloud_assistant_questions). We have generated this dataset using a source document and LlamaIndex's question generator (see appendix about how to generate these questions).\n",
- "\n",
- "If you want to skip this process (it will take several minutes), we have shared the resulting [dataset on Hugging Face](https://huggingface.co/datasets/argilla/customer_assistant).\n",
- "\n",
- "\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "47IBrvmYTKrB"
- },
- "outputs": [],
- "source": [
- "# Read our source questions\n",
- "dataset = load_dataset(\"argilla/cloud_assistant_questions\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "PfcOQ67ZSHZK"
- },
- "outputs": [],
- "source": [
- "# Read and parse the document using Unstructured\n",
- "UnstructuredReader = download_loader(\"UnstructuredReader\", refresh_cache=True)\n",
- "loader = UnstructuredReader()\n",
- "# You can download this doc from: https://huggingface.co/datasets/argilla/cloud_assistant_questions/raw/main/argilla_cloud.txt\n",
- "documents = loader.load_data(\"argilla_cloud.txt\")\n",
- "\n",
- "# Set up the Llama index context\n",
- "gpt_35_context = ServiceContext.from_defaults(\n",
- " llm=OpenAI(model=\"gpt-3.5-turbo\", temperature=0.3)\n",
- ")\n",
- "\n",
- "# Index the document and set up the engine\n",
- "index = VectorStoreIndex.from_documents(documents, service_context=gpt_35_context)\n",
- "query_engine = index.as_query_engine(similarity_top_k=2)\n",
- "\n",
- "contexts = []\n",
- "answers = []\n",
- "questions = dataset[\"train\"][\"question\"]\n",
- "\n",
- "# Inference over the questions\n",
- "for question in tqdm(questions):\n",
- " response = query_engine.query(question)\n",
- " contexts.append([x.node.get_content() for x in response.source_nodes])\n",
- " answers.append(str(response))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "UKRsz0WxUBkI",
- "outputId": "3288bfed-9ccc-4140-973e-cf7f2e265ba2"
- },
- "outputs": [
- {
- "name": "stdout",
- "output_type": "stream",
- "text": [
- "Question: What is the ticketing system used by Argilla for customer support?\n",
- "Answer: The ticketing system used by Argilla for customer support is not specified in the given context information.\n",
- "Context: [\"This process ensures the client administrator has full control over their team's access and can manage their workspace efficiently.Plans The plans for the Argilla Cloud service depend on the volume of records processed, with several tiers available to suit varying needs.Each tier has a corresponding monthly and annual price, with a 10% discount applied to the annual pricing option.The tier selection and associated price will be determined by the client's selection in the Service Order Form section of the Terms of Service document.Plans are: Starter 1 Million records Base 3 Million records Medium 4 Million records Large 6 million records\\n\\nSupport Argilla Cloud offers comprehensive support services to address various issues that may arise during the use of our service.Support levels are categorized into four distinct tiers, based on the severity of the issue, and a separate category for feature requests.The support process, response times, and procedures differ for each category.(1) Critical Issues Critical issues are characterized by: Severe impact on the Service, potentially rendering it completely non-functional.Disruption of critical service operations or functions.Obstruction of entire customer workflows.In the case of a critical issue, Argilla will: Assign specialist(s) to correct the issue on an expedited basis.Provide ongoing communication on the status via email and/or phone, according to the customer's preference.Begin work towards identifying a temporary workaround or fix.(2) Major Issues Major issues involve: Limited functionality of the Service.Service instability with periodic interruptions.Material service interruptions in mission-critical functions.Time-sensitive questions impacting performance or deliverables to end-clients.Upon encountering a major issue, Argilla will: Assign a specialist to begin a resolution.Implement additional, escalated procedures as reasonably determined necessary by Argilla Support Services staff.(3) Minor Issues Minor issues include: Errors causing partial, non-critical functionality loss.The need for clarification on procedures or information in documentation.Errors in service that may impact performance deliverables.(4) Trivial Issues Trivial issues are characterized by: Errors in system development with little to no impact on performance.Feature Requests Feature requests involve: Requesting a product enhancement.For feature requests, Argilla will: Respond regarding the relevance and interest in incorporating the requested feature.In summary, Argilla Cloud's support services are designed to provide timely and efficient assistance for issues of varying severity, ensuring a smooth and reliable user experience.All plans include Monday to Friday during office hours (8am to 17pm CEST) with additional support upon request.The Support Channels and features of each tier are shown below:\\n\\nStarter: Slack Community.Severity 1 - Response time < 4 hours.Severity 2 - Response time < 8 hours.Severity 3 - Response time < 48 hours.Severity 4 not specified.Base: Ticketing System, Severity 1 - Response time < 4 hours.Severity 2 - Response time < 8 hours.Severity 3 - Response time < 24 hours.Severity 4 not specified.Medium: Ticketing System and dedicated Slack channel, Severity 1 - Response time < 4 hours.Severity 2 - Response time < 8 hours.Severity 3 - Response time < 24 hours.Severity 4 one week\\n\\nLarge: Ticketing System and dedicated Slack channel, Severity 1 - Response time < 4 hours.Severity 2 - Response time < 8 hours.Severity 3 - Response time < 24 hours.Severity 4 one week.Data backup and recovery plan Argilla Cloud is committed to ensuring the safety and availability of your data.Our system is designed to run six data backups per day as a standard procedure.These backups capture a snapshot of the system state at the time of the backup, enabling restoration to that point if necessary.Our Recovery Point Objective (RPO) is four hours.This means that in the event of a system failure, the maximum data loss would be up to the last four hours of data input.We achieve this by running regular backups throughout the day, reducing the time window of potential data loss.Our Recovery Time Objective (RTO) is one hour.This is the maximum acceptable length of time that your system could be down following a failure or disruption.It represents our commitment to ensuring that your services are restored as quickly as possible.In the event of a disruption, our team will first evaluate the issue to determine the best course of action.If data recovery is necessary, we will restore from the most recent backup.We will then work to identify and resolve the root cause of the disruption to prevent a recurrence.Finally, we conduct regular test restores to ensure that our backup system is working as intended.These tests verify the integrity of the backup data and the functionality of the restore process.\", \"This documents an overview of the Argilla Cloud service - a comprehensive Software as a Service (SaaS) solution for data labeling and curation.The service is specifically designed to meet the needs of businesses seeking a reliable, secure, and user-friendly platform for data management.The key components of our service include advanced security measures, robust data backup and recovery protocols, flexible pricing options, and dedicated customer support.The onboarding process is efficient, enabling clients to start using the service within one business day.The scope of this proposal includes details on the aforementioned aspects, providing a clear understanding of the service offerings and associated processes.Argilla Cloud offers four plans:\\n\\nStarter: Ideal for teams initiating their journey in scaling data curation and labelling projects.Perfect for environments where production monitoring is not a requirement.Base: Tailored for teams seeking to amplify their data curation, labelling efforts, and model monitoring, with enhanced support from Argilla.Medium: Designed for teams expanding their language model pipelines, requiring robust ML lifecycle management fortified by Argilla's comprehensive support.Large: Geared towards teams heavily dependent on language model pipelines, human feedback, and applications, requiring complete ML lifecycle management with robust support.Scope of services Argilla Cloud, a fully managed SaaS, encompasses the following functionalities: Unrestricted Users, Datasets, and Workspaces: The service imposes no limits on the number of users, datasets, or workspaces, supporting scalability of operations.Role-Based Access Control: Administrators and annotators have differentiated access rights to ensure structured and secure data management.Custom Subdomain: Clients are provided with a distinct argilla.io subdomain for accessing the platform.Regular Updates and Upgrades: The service includes regular platform patches and upgrades as part of routine maintenance to uphold system integrity and security.Managed Service: Infrastructure maintenance, backend operations, and other technical aspects are managed by Argilla, eliminating the need for client-side management.Security The security framework of the Argilla Cloud service involves a multi-faceted approach: Data Encryption at Rest: Data stored within the system is encrypted, forming a crucial layer of security.This process automatically encrypts data prior to storage, guarding against unauthorized access.Network Security Measures: The infrastructure has been designed to prevent unauthorized intrusion and to ensure consistent service availability.Measures include firewall protections, intrusion detection systems, and scheduled vulnerability scans to detect and address potential threats.Role-Based Access Control: The system implements role-based access control, defining access levels based on user roles.This mechanism controls the extent of access to sensitive information, aligning it with the responsibilities of each role.Security Audits: Regular audits of security systems and protocols are conducted to detect potential vulnerabilities and verify adherence to security standards.Employee Training: All personnel receive regular security training, fostering an understanding of the latest threats and the importance of security best practices.Incident Response Protocol: In the case of a security incident, a pre-defined incident response plan is activated.This plan outlines the procedures for managing different types of security events, and aims to ensure swift mitigation of potential damage.In summary, the security measures in place include data encryption, network security protocols, role-based access control, regular audits, employee training, and a comprehensive incident response plan.These measures contribute to a secure environment for data management.Setup and onboarding The process for setup and onboarding for Argilla Cloud is designed to be efficient and straightforward.The procedure involves a sequence of steps to ensure a smooth transition and optimal use of the service.Step 1: Account Creation The setup process begins with the creation of the client owner account.We require the client to provide the following details: Full name of the administrator Preferred username Administrator's email address Once these details are received, we send an onboarding email to sign up.Step 2: Platform Orientation Once logged in, the administrator has full access to the Argilla Cloud platform.They can familiarize themselves with the platform interface and various features.If required, a guided tour or tutorial can be provided to walk the administrator through the platform.Step 3: User Management The administrator is then responsible for setting up additional user accounts.They can invite users via email, manage roles (admin, annotator, etc.), and assign access permissions to different workspaces and datasets.Step 4: Workspace and Dataset Configuration The administrator can create and manage multiple workspaces and datasets.They have the option to configure settings as per their team's requirements, including assigning datasets to specific workspaces and managing access permissions.Step 5: Training and Support Argilla provides open resources and support to aid in the onboarding process.This includes user manuals, tutorials, and access to our support team for any queries or issues that may arise during the setup and onboarding process.By following these steps, new users can be quickly onboarded and begin using the Argilla Cloud service with minimal downtime.\"]\n"
- ]
- }
- ],
- "source": [
- "# Show an example of q, a, and context\n",
- "print(f\"Question: {questions[0]}\")\n",
- "print(f\"Answer: {answers[0]}\")\n",
- "print(f\"Context: {contexts[0]}\")"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "y6Bavj4Abkwa"
- },
- "source": [
- "## Create Argilla dataset and collect feedback\n",
- "\n",
- "\n",
- "We set up an Argilla Dataset for gathering human feedback.\n",
- "\n",
- "\n",
- "For fine-tuning, we need to set up a text question to gather the human written or edited responses. This data is known as completion or demonstration data.\n",
- "\n",
- "Additionally, leveraging the multi-aspect feedback capabilities of Argilla, we set up two additional feedback dimensions to rate the relevance of the question (as they're synthetic they might be irrelevant or bad quality) and the quality of the context retrieved from our retriever component (can be used to improve the RAG configuration).\n",
- "\n"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "ID3Qu9aVbtaX"
- },
- "outputs": [],
- "source": [
- "dataset = rg.FeedbackDataset(\n",
- " fields=[rg.TextField(name=\"user-message\"), rg.TextField(name=\"context\")],\n",
- " questions=[\n",
- " rg.RatingQuestion(name=\"question-rating\", title=\"Rate the relevance of the user question\", values=[1,2,3,4,5], required=False),\n",
- " rg.RatingQuestion(name=\"context-rating\", title=\"Rate the quality and relevancy of context for the assistant\", values=[1,2,3,4,5], required=False),\n",
- " rg.TextQuestion(name=\"response\", title=\"Write a helpful, harmless, accurate response to the user question\"),\n",
- " ]\n",
- ")"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "btwbDF6_PayX"
- },
- "source": [
- "We use the questions, context, and generated responses to build our feedback records. We pre-fill the responses in the UI with OpenAI's responses using `suggestions` and ask our labelers to edit them if necessary."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "bW63w95KcfwS"
- },
- "outputs": [],
- "source": [
- "records = []\n",
- "\n",
- "for question, answer, context in tqdm(zip(questions, answers, contexts), total=len(questions)):\n",
- " # Instantiate the FeedbackRecord\n",
- " feedback_record = rg.FeedbackRecord(\n",
- " fields={\"user-message\": question, \"context\": \"\\n\".join(context)},\n",
- " suggestions=[\n",
- " {\n",
- " \"question_name\": \"response\",\n",
- " \"value\": answer,\n",
- " }\n",
- " ]\n",
- " )\n",
- " records.append(feedback_record)\n",
- "\n",
- "# Publish dataset in Argilla UI\n",
- "dataset = dataset.push_to_argilla(name=\"customer_assistant\", workspace=\"admin\")\n",
- "dataset.add_records(records)\n",
- "\n",
- "# Optional: store and version dataset in the Hub\n",
- "#dataset = dataset.push_to_huggingface(\"argilla/rg_customer_assistant\")"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now, the dataset is available for collecting feedback with the Argilla UI. Here's a video showing the workflow for labelers:\n",
- "\n",
- ""
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "9yLxmSDsQPts"
- },
- "source": [
- "## Prepare Argilla dataset for fine-tuning\n",
- "\n",
- "\n",
- "We now read the responses from Argilla and prepare the dataset for fine-tuning following the [fine-tuning format from OpenAI guides](https://platform.openai.com/docs/guides/fine-tuning).\n",
- "\n",
- "We use the quick adaptation of LlamaIndex's `TEXT_QA_PROMPT` system prompt and the fine-tuned responses from our Argilla dataset."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Read the dataset from Argilla\n",
- "dataset = rg.FeedbackDataset.from_argilla(\"customer_assistant\", workspace=\"admin\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If you have skipped the previous steps run this to get the pre-built dataset.\n",
- "\n",
- "```python\n",
- "dataset = rg.FeedbackDataset.from_huggingface(\"argilla/customer_assistant\")\n",
- "```"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "N5hFLJaHSz9C"
- },
- "outputs": [],
- "source": [
- "# Adaptation from LlamaIndex's TEXT_QA_PROMPT_TMPL_MSGS[1].content\n",
- "user_message_prompt =\"\"\"Context information is below.\n",
- "---------------------\n",
- "{context_str}\n",
- "---------------------\n",
- "Given the context information and not prior knowledge but keeping your Argilla Cloud assistant style, answer the query.\n",
- "Query: {query_str}\n",
- "Answer:\n",
- "\"\"\"\n",
- "# Adaptation from LlamaIndex's TEXT_QA_SYSTEM_PROMPT\n",
- "system_prompt = \"\"\"You are an expert customer service assistant for the Argilla Cloud product that is trusted around the world.\n",
- "Always answer the query using the provided context information, and not prior knowledge.\n",
- "Some rules to follow:\n",
- "1. Never directly reference the given context in your answer.\n",
- "2. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines.\n",
- "\"\"\""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "siPKLnctRLpD"
- },
- "outputs": [],
- "source": [
- "def formatting_func(sample: dict) -> Union[Tuple[str, str, str, str], List[Tuple[str, str, str, str]]]:\n",
- " from uuid import uuid4\n",
- " if sample[\"response\"]:\n",
- " chat = str(uuid4())\n",
- " user_message = user_message_prompt.format(context_str=sample[\"context\"], query_str=sample[\"user-message\"])\n",
- " return [\n",
- " (chat, \"0\", \"system\", system_prompt),\n",
- " (chat, \"1\", \"user\", user_message),\n",
- " (chat, \"2\", \"assistant\", sample[\"response\"][0][\"value\"])\n",
- " ]\n",
- "\n",
- "task = TrainingTask.for_chat_completion(formatting_func=formatting_func)"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "SVvdU9EqVWSR"
- },
- "source": [
- "## Fine-tune GPT3.5 with high-quality feedback\n",
- "\n",
- "\n",
- "We fine-tune `gpt-3.5-turbo` with the exported dataset using the Argilla Trainer."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "afLyk289VhBL"
- },
- "outputs": [],
- "source": [
- "trainer = ArgillaTrainer(\n",
- " dataset=dataset,\n",
- " task=task,\n",
- " framework=\"openai\",\n",
- ")\n",
- "trainer.train(output_dir=\"my-ft-openai-model\")"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "eG-zXJJ2dbi_"
- },
- "source": [
- "## Evaluating base vs fine-tuned with human preference data\n",
- "\n",
- "We set up a new feedback dataset for gathering human feedback to evaluate the fine-tuned model against the base model, using the test dataset.\n",
- "\n",
- "There are many ways to collect feedback for this phase. The most suitable in this case is human preference data over responses from the two models: *asking our labelers which response is the most accurate and helpful*. We can easily do this with Argilla's `RankingQuestion`.\n",
- "\n",
- "Additionally, as both responses can be equally bad, we can ask labelers to write down a correct response. In this case, we would be collecting demonstration data to add to our fine-tuning workflow.\n",
- "\n",
- "\n",
- "### Create dataset and collect feedback\n",
- "\n",
- "We set up and publish a new dataset with a `RankingQuestion` and `TextQuestion`, showing our labelers the `user-message` and two responses (from the base and the fine-tuned models)."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "tsQT0Ps1igZc"
- },
- "outputs": [],
- "source": [
- "dataset = rg.FeedbackDataset(\n",
- " fields=[rg.TextField(name=\"user-message\"), rg.TextField(name=\"response-a\"), rg.TextField(name=\"response-b\")],\n",
- " questions=[\n",
- " rg.RankingQuestion(name=\"preference\", title=\"Which response is more helpful, harmless, and accurate.\", values=[\"response-a\", \"response-b\"]),\n",
- " rg.TextQuestion(name=\"response\", title=\"If none is good, write a helpful, harmless, accurate response to the user question\", required=False),\n",
- " ]\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "95-iBx8fgoGg"
- },
- "outputs": [],
- "source": [
- "# Read our test questions\n",
- "questions = load_dataset(\"argilla/cloud_assistant_questions\", split=\"test\")[\"question\"]"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "background_save": true
- },
- "id": "NPPLoZGae9ia"
- },
- "outputs": [],
- "source": [
- "# Generate responses with base model\n",
- "index = VectorStoreIndex.from_documents(documents, service_context=gpt_35_context)\n",
- "query_engine = index.as_query_engine(similarity_top_k=2)\n",
- "\n",
- "contexts = []\n",
- "base_model_responses = []\n",
- "\n",
- "for question in tqdm(questions):\n",
- " response = query_engine.query(question)\n",
- " base_model_responses.append(str(response))"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "AU9NwKnQgAPu"
- },
- "outputs": [],
- "source": [
- "# Generate responses with ft model: replace with the id of your ft model\n",
- "ft_context = ServiceContext.from_defaults(\n",
- " llm=OpenAI(model=\"ft:gpt-3.5-turbo-...\", temperature=0.3)\n",
- ")\n",
- "index = VectorStoreIndex.from_documents(documents, service_context=ft_context)\n",
- "query_engine = index.as_query_engine(similarity_top_k=2)\n",
- "\n",
- "contexts = []\n",
- "ft_model_responses = []\n",
- "\n",
- "for question in tqdm(questions):\n",
- " response = query_engine.query(question)\n",
- " ft_model_responses.append(str(response))"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "hsPoYfgJTCOD"
- },
- "source": [
- "An important step here is to randomize the order in which responses are shown.\n",
- "\n",
- "If we show the fine-tuned model response always as the first option, we can introduce position bias (labelers always choosing a certain position) or make it evident to users that there are two obviously different models.\n",
- "\n",
- "To avoid this, we randomize the position and keep two metadata fields indicating which model has produced `response-a` and `response-b`. When collecting the responses, we'll use this metadata to map the ranking with each model."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "0zvH3sDkhMFJ"
- },
- "outputs": [],
- "source": [
- "records = []\n",
- "for base, ft, question in zip(base_model_responses, ft_model_responses, questions):\n",
- " # Randomizing the position is a highly important step to mitigate labeler biases\n",
- " # Shuffle the order of base and ft\n",
- " response_a, response_b = random.sample([base, ft], 2)\n",
- "\n",
- " # Map the responses back to their model names\n",
- " models = {\n",
- " base: \"base_model\",\n",
- " ft: \"ft_model\"\n",
- " }\n",
- " feedback_record = rg.FeedbackRecord(\n",
- " fields={\"user-message\": question, \"response-a\": response_a, \"response-b\": response_b},\n",
- " metadata={\"response-a-model\": models[response_a], \"response-b-model\": models[response_b]}\n",
- " )\n",
- "\n",
- " records.append(feedback_record)\n",
- "\n",
- "dataset = dataset.push_to_argilla(name=\"finetuned-vs-base-preference\", workspace=\"admin\")\n",
- "dataset.add_records(records)"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now, the dataset is available for collecting feedback with the Argilla UI. Here's a video showing the workflow for labelers:\n",
- "\n",
- ""
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "eRiVf39Bblmh"
- },
- "source": [
- "### Retrieve and analyze responses\n",
- "\n",
- "We can dynamically collect the responses from our labelers. In this case, we will compute the win rate and ties (as users can indicate both responses are equally good or bad).\n",
- "\n",
- "For the tutorial, we only have one user but Argilla Feedback is fully multi-user, which means you can collect feedback from several users for each data point, increasing the quality of the evaluation. \n",
- "\n",
- "You can read more about multi-user scenarios and built-in unification methods [on this guide](/practical_guides/collect_responses.md).\n",
- "\n",
- "With a very small evaluation set, we can see that the fine-tuned model responses are preferred ~60% of the time, 3x over the base model, and they are both equally good or bad ~20% of the time. \n",
- "\n",
- "\n",
- "Even with a very small fine-tuning and evaluation dataset, this already shows **promising benefits of fine-tuning models for enhancing RAG systems**.\n",
- "\n",
- "\n",
- ""
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "XZKBhlKCbwUV"
- },
- "outputs": [],
- "source": [
- "# Retrieve the dataset from Argilla\n",
- "dataset = rg.FeedbackDataset.from_argilla(name=\"finetuned-vs-base-preference\", workspace=\"admin\")\n",
- "\n",
- "win_rates = {\n",
- " 'ft_model': 0,\n",
- " 'base_model': 0,\n",
- " 'tie': 0\n",
- "}\n",
- "\n",
- "# Compute the win and tie rates\n",
- "for record in dataset.records:\n",
- " if len(record.responses) > 0:\n",
- " for response in record.responses:\n",
- " model_a = record.metadata[\"response-a-model\"]\n",
- " model_b = record.metadata[\"response-b-model\"]\n",
- " preference = response.values['preference'].value\n",
- " if preference[0].rank > preference[1].rank:\n",
- " win_rates[model_a] = win_rates[model_a] + 1\n",
- " elif preference[1].rank > preference[0].rank:\n",
- " win_rates[model_b] = win_rates[model_b] + 1\n",
- " else:\n",
- " win_rates['tie'] = win_rates['tie'] + 1\n",
- "win_rates\n",
- "# {'ft_model': 17, 'base_model': 6, 'tie': 5}"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "BpoH-OnRnuoL"
- },
- "outputs": [],
- "source": [
- "# Let's make the labels more explicit\n",
- "data = {'gpt3.5-fine-tuned': 17, 'gpt3.5-base': 6, 'tie': 5}\n",
- "total = sum(data.values())\n",
- "\n",
- "# Calculate percentages\n",
- "percentages = [value / total * 100 for value in data.values()]\n",
- "\n",
- "# Settings\n",
- "colors = ['blue', 'grey', 'black']\n",
- "labels = [f\"{key} ({value:.2f}%)\" for key, value in zip(data.keys(), percentages)]\n",
- "\n",
- "# Plotting\n",
- "plt.figure(figsize=(12, 2))\n",
- "\n",
- "# The cumulative percentage is used to shift the starting point of each subsequent segment\n",
- "cumulative_percentages = 0\n",
- "\n",
- "for percent, color, label in zip(percentages, colors, labels):\n",
- " plt.barh('Models', percent, color=color, label=label, left=cumulative_percentages)\n",
- " plt.text(cumulative_percentages + percent/2, 0, label, ha='center', va='center', color='white', fontsize=10)\n",
- " cumulative_percentages += percent\n",
- "\n",
- "plt.gca().axes.get_yaxis().set_visible(False)\n",
- "plt.xlim(0, 100)\n",
- "plt.title('Model Win Rates')\n",
- "plt.legend(loc=\"upper center\", bbox_to_anchor=(0.5, -0.25), ncol=3)\n",
- "plt.tight_layout()\n",
- "\n",
- "# Display\n",
- "plt.show()"
- ]
- },
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "hCrnoTw_GIyr"
- },
- "source": [
- "## Appendix: Generating questions with Llama Index\n",
- "\n",
- "\n",
- "We use the `DatasetGenerator` from Llama Index to generate a set of questions using a document about Argilla Cloud."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "kf-hZurHckDf"
- },
- "outputs": [],
- "source": [
- "UnstructuredReader = download_loader(\"UnstructuredReader\", refresh_cache=True)\n",
- "loader = UnstructuredReader()\n",
- "\n",
- "# You can download this doc from: https://huggingface.co/datasets/argilla/cloud_assistant_questions/raw/main/argilla_cloud.txt\n",
- "documents = loader.load_data(\"argilla_cloud.txt\")\n",
- "\n",
- "gpt_35_context = ServiceContext.from_defaults(\n",
- " llm=OpenAI(model=\"gpt-3.5-turbo\", temperature=0.4),\n",
- " chunk_size=60\n",
- ")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "wSVCTgEsc1zm"
- },
- "outputs": [],
- "source": [
- "question_gen_query = (\n",
- " \"You are customer support and sales expert of Argilla. Your task is to setup \"\n",
- " \"a set of frequently asked questions about the Argilla Cloud service, offer and plans\"\n",
- " \"formulate a single question that could be asked by a potential B2B client interested in Argilla Cloud \"\n",
- " \". Restrict the question to the context information provided and don't ask general questions not related to the service and the context provided.\"\n",
- ")\n",
- "\n",
- "dataset_generator = DatasetGenerator.from_documents(\n",
- " documents,\n",
- " question_gen_query=question_gen_query,\n",
- " service_context=gpt_35_context,\n",
- " num_questions_per_chunk=100\n",
- ")\n",
- "\n",
- "questions = dataset_generator.generate_questions_from_nodes(num=300)"
- ]
- }
- ],
- "metadata": {
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "FOyRtBFaD8Ti"
+ },
+ "source": [
+ "# 🪄 Fine-tuning and evaluating GPT-3.5 with human feedback for RAG \n",
+ "\n",
+ "This guide explains how to fine-tune OpenAI's GPT3.5-turbo with your own data and Argilla to improve a RAG (Retrieval Augmented Generation) system. \n",
+ "\n",
+ "It includes the following steps:\n",
+ "\n",
+ "- Setting up a RAG pipeline using [LlamaIndex](https://github.com/jerryjliu/llama_index) and [Unstructured](https://github.com/Unstructured-IO/unstructured) to answer questions using a document about Argilla Cloud.\n",
+ "- Generating potential questions with LlamaIndex to build a training and test set.\n",
+ "- Building a dataset for collecting human written responses with Argilla.\n",
+ "- Fine-tuning GPT3.5-turbo with high-quality data.\n",
+ "- Evaluating the fine-tuned model vs. the base model with human preference data from Argilla.\n",
+ "\n",
+ "The goal of the tutorial is to demonstrate how to incorporate human feedback into your LLM development for two critical stages: \n",
+ "\n",
+ "1. Gathering **high-quality data for fine-tuning**, \n",
+ "2. Gathering **human feedback for evaluation of LLM applications**.\n",
+ "\n",
+ "\n",
+ "Given the ongoing debate between Retrieval Augmented Generation (RAG) and fine-tuning, we selected a real-world RAG use case to demonstrate how fine-tuning enhances the style, utility, and relevance of responses within a RAG application. The resulting system will be a Hybrid RAG system (RAG using fine-tuned models) as [described in this article](https://towardsdatascience.com/rag-vs-finetuning-which-is-the-best-tool-to-boost-your-llm-application-94654b1eaba7). \n",
+ "\n",
+ "The screenshot below displays the evaluation dataset, termed the \"human preference dataset.\" In it, `response-a` is produced by the fine-tuned model, while `response-b` comes from the base GPT-3.5 model. With just minor fine-tuning and without altering the system message, we've directed the LLM's behavior towards generating responses that are more helpful, faithful, friendly, and aligned with our brand.\n",
+ "\n",
+ "Fine-tuning effectively mitigates common RAG challenges, like the LLM referring to the context using phrases such as \"The context does not provide information about this.\" This enhancement is notable even when we had incorporated directives in the system message to deter such references, like \"2. Avoid phrases such as 'Based on the context, ...' or 'The context information ...'.\" (see Llama Index default prompt later).\n",
+ "\n",
+ "You can also browse the [datasets hosted with Argilla Hugging Face Spaces](https://huggingface.co/spaces/argilla/fine-tune-chat-gpt). User and password: argilla / 12345678. The dataset for this stage is `customer-assistant` and for the evaluation step is `finetuned-vs-base-preference`.\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "By the end of the tutorial, you'll be using a fine-tuned model for RAG and have a human evaluation workflow in place to continuously evaluate your LLM application (see below for a comparison of the base gpt3.5 vs. the fine-tuned gpt3.5 for this application).\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n",
+ "Let's get started!\n",
+ "\n",
+ "## Setup\n",
+ "\n",
+ "To run this tutorial, you need to [install and launch Argilla](https://docs.v1.argilla.io/en/latest/getting_started/quickstart_installation.html), as well as some other packages."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "j76dPBrN3iR_"
+ },
+ "outputs": [],
+ "source": [
+ "%pip install argilla openai datasets llama-index unstructured -qqq"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "RI_lnoziIkww"
+ },
+ "outputs": [],
+ "source": [
+ "# Import the needed libraries\n",
+ "import os\n",
+ "import random\n",
+ "from tqdm import tqdm\n",
+ "import matplotlib.pyplot as plt\n",
+ "\n",
+ "import openai\n",
+ "\n",
+ "import argilla as rg\n",
+ "from argilla.feedback import TrainingTask\n",
+ "from argilla.feedback import ArgillaTrainer\n",
+ "\n",
+ "from typing import Union, Tuple, List\n",
+ "\n",
+ "from llama_index.core import ServiceContext, VectorStoreIndex, download_loader\n",
+ "from llama_index.llms.openai import OpenAI\n",
+ "from llama_index.core.evaluation import DatasetGenerator\n",
+ "\n",
+ "from datasets import load_dataset"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the URL and API_KEY:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "kIrT-iWx3kbX"
+ },
+ "outputs": [],
+ "source": [
+ "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
+ "# Replace api_key if you configured a custom API key\n",
+ "# Replace workspace with the name of your workspace\n",
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"owner.apikey\", workspace=\"admin\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Set the HF_TOKEN environment variable\n",
+ "# import os\n",
+ "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
+ "\n",
+ "# # Replace api_url with the url to your HF Spaces URL\n",
+ "# # Replace api_key if you configured a custom API key\n",
+ "# # Replace workspace with the name of your workspace\n",
+ "# rg.init(\n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
+ "# api_key=\"owner.apikey\",\n",
+ "# workspace=\"admin\",\n",
+ "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
+ "# )"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "T45WtJ9t9Vzq"
+ },
+ "outputs": [],
+ "source": [
+ "# Your openAI key is needed for generation and fine-tuning\n",
+ "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n",
+ "openai.api_key = os.environ[\"OPENAI_API_KEY\"]"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Enable Telemetry\n",
+ "\n",
+ "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../../reference/telemetry.md) page.\n",
+ "\n",
+ "```python\n",
+ "from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
+ "tutorial_running()\n",
+ "```"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "K4ExAXiKSCu5"
+ },
+ "source": [
+ "## Generating responses with LlamaIndex and GPT3.5\n",
+ "\n",
+ "We generate responses for the generated questions using [this dataset about Argilla Cloud](https://huggingface.co/datasets/argilla/cloud_assistant_questions). We have generated this dataset using a source document and LlamaIndex's question generator (see appendix about how to generate these questions).\n",
+ "\n",
+ "If you want to skip this process (it will take several minutes), we have shared the resulting [dataset on Hugging Face](https://huggingface.co/datasets/argilla/customer_assistant).\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "47IBrvmYTKrB"
+ },
+ "outputs": [],
+ "source": [
+ "# Read our source questions\n",
+ "dataset = load_dataset(\"argilla/cloud_assistant_questions\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "PfcOQ67ZSHZK"
+ },
+ "outputs": [],
+ "source": [
+ "# Read and parse the document using Unstructured\n",
+ "UnstructuredReader = download_loader(\"UnstructuredReader\", refresh_cache=True)\n",
+ "loader = UnstructuredReader()\n",
+ "# You can download this doc from: https://huggingface.co/datasets/argilla/cloud_assistant_questions/raw/main/argilla_cloud.txt\n",
+ "documents = loader.load_data(\"argilla_cloud.txt\")\n",
+ "\n",
+ "# Set up the Llama index context\n",
+ "gpt_35_context = ServiceContext.from_defaults(\n",
+ " llm=OpenAI(model=\"gpt-3.5-turbo\", temperature=0.3)\n",
+ ")\n",
+ "\n",
+ "# Index the document and set up the engine\n",
+ "index = VectorStoreIndex.from_documents(documents, service_context=gpt_35_context)\n",
+ "query_engine = index.as_query_engine(similarity_top_k=2)\n",
+ "\n",
+ "contexts = []\n",
+ "answers = []\n",
+ "questions = dataset[\"train\"][\"question\"]\n",
+ "\n",
+ "# Inference over the questions\n",
+ "for question in tqdm(questions):\n",
+ " response = query_engine.query(question)\n",
+ " contexts.append([x.node.get_content() for x in response.source_nodes])\n",
+ " answers.append(str(response))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
"colab": {
- "provenance": []
- },
- "kernelspec": {
- "display_name": "Python 3",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.10.13"
+ "base_uri": "https://localhost:8080/"
+ },
+ "id": "UKRsz0WxUBkI",
+ "outputId": "3288bfed-9ccc-4140-973e-cf7f2e265ba2"
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Question: What is the ticketing system used by Argilla for customer support?\n",
+ "Answer: The ticketing system used by Argilla for customer support is not specified in the given context information.\n",
+ "Context: [\"This process ensures the client administrator has full control over their team's access and can manage their workspace efficiently.Plans The plans for the Argilla Cloud service depend on the volume of records processed, with several tiers available to suit varying needs.Each tier has a corresponding monthly and annual price, with a 10% discount applied to the annual pricing option.The tier selection and associated price will be determined by the client's selection in the Service Order Form section of the Terms of Service document.Plans are: Starter 1 Million records Base 3 Million records Medium 4 Million records Large 6 million records\\n\\nSupport Argilla Cloud offers comprehensive support services to address various issues that may arise during the use of our service.Support levels are categorized into four distinct tiers, based on the severity of the issue, and a separate category for feature requests.The support process, response times, and procedures differ for each category.(1) Critical Issues Critical issues are characterized by: Severe impact on the Service, potentially rendering it completely non-functional.Disruption of critical service operations or functions.Obstruction of entire customer workflows.In the case of a critical issue, Argilla will: Assign specialist(s) to correct the issue on an expedited basis.Provide ongoing communication on the status via email and/or phone, according to the customer's preference.Begin work towards identifying a temporary workaround or fix.(2) Major Issues Major issues involve: Limited functionality of the Service.Service instability with periodic interruptions.Material service interruptions in mission-critical functions.Time-sensitive questions impacting performance or deliverables to end-clients.Upon encountering a major issue, Argilla will: Assign a specialist to begin a resolution.Implement additional, escalated procedures as reasonably determined necessary by Argilla Support Services staff.(3) Minor Issues Minor issues include: Errors causing partial, non-critical functionality loss.The need for clarification on procedures or information in documentation.Errors in service that may impact performance deliverables.(4) Trivial Issues Trivial issues are characterized by: Errors in system development with little to no impact on performance.Feature Requests Feature requests involve: Requesting a product enhancement.For feature requests, Argilla will: Respond regarding the relevance and interest in incorporating the requested feature.In summary, Argilla Cloud's support services are designed to provide timely and efficient assistance for issues of varying severity, ensuring a smooth and reliable user experience.All plans include Monday to Friday during office hours (8am to 17pm CEST) with additional support upon request.The Support Channels and features of each tier are shown below:\\n\\nStarter: Slack Community.Severity 1 - Response time < 4 hours.Severity 2 - Response time < 8 hours.Severity 3 - Response time < 48 hours.Severity 4 not specified.Base: Ticketing System, Severity 1 - Response time < 4 hours.Severity 2 - Response time < 8 hours.Severity 3 - Response time < 24 hours.Severity 4 not specified.Medium: Ticketing System and dedicated Slack channel, Severity 1 - Response time < 4 hours.Severity 2 - Response time < 8 hours.Severity 3 - Response time < 24 hours.Severity 4 one week\\n\\nLarge: Ticketing System and dedicated Slack channel, Severity 1 - Response time < 4 hours.Severity 2 - Response time < 8 hours.Severity 3 - Response time < 24 hours.Severity 4 one week.Data backup and recovery plan Argilla Cloud is committed to ensuring the safety and availability of your data.Our system is designed to run six data backups per day as a standard procedure.These backups capture a snapshot of the system state at the time of the backup, enabling restoration to that point if necessary.Our Recovery Point Objective (RPO) is four hours.This means that in the event of a system failure, the maximum data loss would be up to the last four hours of data input.We achieve this by running regular backups throughout the day, reducing the time window of potential data loss.Our Recovery Time Objective (RTO) is one hour.This is the maximum acceptable length of time that your system could be down following a failure or disruption.It represents our commitment to ensuring that your services are restored as quickly as possible.In the event of a disruption, our team will first evaluate the issue to determine the best course of action.If data recovery is necessary, we will restore from the most recent backup.We will then work to identify and resolve the root cause of the disruption to prevent a recurrence.Finally, we conduct regular test restores to ensure that our backup system is working as intended.These tests verify the integrity of the backup data and the functionality of the restore process.\", \"This documents an overview of the Argilla Cloud service - a comprehensive Software as a Service (SaaS) solution for data labeling and curation.The service is specifically designed to meet the needs of businesses seeking a reliable, secure, and user-friendly platform for data management.The key components of our service include advanced security measures, robust data backup and recovery protocols, flexible pricing options, and dedicated customer support.The onboarding process is efficient, enabling clients to start using the service within one business day.The scope of this proposal includes details on the aforementioned aspects, providing a clear understanding of the service offerings and associated processes.Argilla Cloud offers four plans:\\n\\nStarter: Ideal for teams initiating their journey in scaling data curation and labelling projects.Perfect for environments where production monitoring is not a requirement.Base: Tailored for teams seeking to amplify their data curation, labelling efforts, and model monitoring, with enhanced support from Argilla.Medium: Designed for teams expanding their language model pipelines, requiring robust ML lifecycle management fortified by Argilla's comprehensive support.Large: Geared towards teams heavily dependent on language model pipelines, human feedback, and applications, requiring complete ML lifecycle management with robust support.Scope of services Argilla Cloud, a fully managed SaaS, encompasses the following functionalities: Unrestricted Users, Datasets, and Workspaces: The service imposes no limits on the number of users, datasets, or workspaces, supporting scalability of operations.Role-Based Access Control: Administrators and annotators have differentiated access rights to ensure structured and secure data management.Custom Subdomain: Clients are provided with a distinct argilla.io subdomain for accessing the platform.Regular Updates and Upgrades: The service includes regular platform patches and upgrades as part of routine maintenance to uphold system integrity and security.Managed Service: Infrastructure maintenance, backend operations, and other technical aspects are managed by Argilla, eliminating the need for client-side management.Security The security framework of the Argilla Cloud service involves a multi-faceted approach: Data Encryption at Rest: Data stored within the system is encrypted, forming a crucial layer of security.This process automatically encrypts data prior to storage, guarding against unauthorized access.Network Security Measures: The infrastructure has been designed to prevent unauthorized intrusion and to ensure consistent service availability.Measures include firewall protections, intrusion detection systems, and scheduled vulnerability scans to detect and address potential threats.Role-Based Access Control: The system implements role-based access control, defining access levels based on user roles.This mechanism controls the extent of access to sensitive information, aligning it with the responsibilities of each role.Security Audits: Regular audits of security systems and protocols are conducted to detect potential vulnerabilities and verify adherence to security standards.Employee Training: All personnel receive regular security training, fostering an understanding of the latest threats and the importance of security best practices.Incident Response Protocol: In the case of a security incident, a pre-defined incident response plan is activated.This plan outlines the procedures for managing different types of security events, and aims to ensure swift mitigation of potential damage.In summary, the security measures in place include data encryption, network security protocols, role-based access control, regular audits, employee training, and a comprehensive incident response plan.These measures contribute to a secure environment for data management.Setup and onboarding The process for setup and onboarding for Argilla Cloud is designed to be efficient and straightforward.The procedure involves a sequence of steps to ensure a smooth transition and optimal use of the service.Step 1: Account Creation The setup process begins with the creation of the client owner account.We require the client to provide the following details: Full name of the administrator Preferred username Administrator's email address Once these details are received, we send an onboarding email to sign up.Step 2: Platform Orientation Once logged in, the administrator has full access to the Argilla Cloud platform.They can familiarize themselves with the platform interface and various features.If required, a guided tour or tutorial can be provided to walk the administrator through the platform.Step 3: User Management The administrator is then responsible for setting up additional user accounts.They can invite users via email, manage roles (admin, annotator, etc.), and assign access permissions to different workspaces and datasets.Step 4: Workspace and Dataset Configuration The administrator can create and manage multiple workspaces and datasets.They have the option to configure settings as per their team's requirements, including assigning datasets to specific workspaces and managing access permissions.Step 5: Training and Support Argilla provides open resources and support to aid in the onboarding process.This includes user manuals, tutorials, and access to our support team for any queries or issues that may arise during the setup and onboarding process.By following these steps, new users can be quickly onboarded and begin using the Argilla Cloud service with minimal downtime.\"]\n"
+ ]
}
+ ],
+ "source": [
+ "# Show an example of q, a, and context\n",
+ "print(f\"Question: {questions[0]}\")\n",
+ "print(f\"Answer: {answers[0]}\")\n",
+ "print(f\"Context: {contexts[0]}\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "y6Bavj4Abkwa"
+ },
+ "source": [
+ "## Create Argilla dataset and collect feedback\n",
+ "\n",
+ "\n",
+ "We set up an Argilla Dataset for gathering human feedback.\n",
+ "\n",
+ "\n",
+ "For fine-tuning, we need to set up a text question to gather the human written or edited responses. This data is known as completion or demonstration data.\n",
+ "\n",
+ "Additionally, leveraging the multi-aspect feedback capabilities of Argilla, we set up two additional feedback dimensions to rate the relevance of the question (as they're synthetic they might be irrelevant or bad quality) and the quality of the context retrieved from our retriever component (can be used to improve the RAG configuration).\n",
+ "\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "ID3Qu9aVbtaX"
+ },
+ "outputs": [],
+ "source": [
+ "dataset = rg.FeedbackDataset(\n",
+ " fields=[rg.TextField(name=\"user-message\"), rg.TextField(name=\"context\")],\n",
+ " questions=[\n",
+ " rg.RatingQuestion(\n",
+ " name=\"question-rating\",\n",
+ " title=\"Rate the relevance of the user question\",\n",
+ " values=[1, 2, 3, 4, 5],\n",
+ " required=False,\n",
+ " ),\n",
+ " rg.RatingQuestion(\n",
+ " name=\"context-rating\",\n",
+ " title=\"Rate the quality and relevancy of context for the assistant\",\n",
+ " values=[1, 2, 3, 4, 5],\n",
+ " required=False,\n",
+ " ),\n",
+ " rg.TextQuestion(\n",
+ " name=\"response\",\n",
+ " title=\"Write a helpful, harmless, accurate response to the user question\",\n",
+ " ),\n",
+ " ],\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "btwbDF6_PayX"
+ },
+ "source": [
+ "We use the questions, context, and generated responses to build our feedback records. We pre-fill the responses in the UI with OpenAI's responses using `suggestions` and ask our labelers to edit them if necessary."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "bW63w95KcfwS"
+ },
+ "outputs": [],
+ "source": [
+ "records = []\n",
+ "\n",
+ "for question, answer, context in tqdm(\n",
+ " zip(questions, answers, contexts), total=len(questions)\n",
+ "):\n",
+ " # Instantiate the FeedbackRecord\n",
+ " feedback_record = rg.FeedbackRecord(\n",
+ " fields={\"user-message\": question, \"context\": \"\\n\".join(context)},\n",
+ " suggestions=[\n",
+ " {\n",
+ " \"question_name\": \"response\",\n",
+ " \"value\": answer,\n",
+ " }\n",
+ " ],\n",
+ " )\n",
+ " records.append(feedback_record)\n",
+ "\n",
+ "# Publish dataset in Argilla UI\n",
+ "dataset = dataset.push_to_argilla(name=\"customer_assistant\", workspace=\"admin\")\n",
+ "dataset.add_records(records)\n",
+ "\n",
+ "# Optional: store and version dataset in the Hub\n",
+ "# dataset = dataset.push_to_huggingface(\"argilla/rg_customer_assistant\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now, the dataset is available for collecting feedback with the Argilla UI. Here's a video showing the workflow for labelers:\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "9yLxmSDsQPts"
+ },
+ "source": [
+ "## Prepare Argilla dataset for fine-tuning\n",
+ "\n",
+ "\n",
+ "We now read the responses from Argilla and prepare the dataset for fine-tuning following the [fine-tuning format from OpenAI guides](https://platform.openai.com/docs/guides/fine-tuning).\n",
+ "\n",
+ "We use the quick adaptation of LlamaIndex's `TEXT_QA_PROMPT` system prompt and the fine-tuned responses from our Argilla dataset."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Read the dataset from Argilla\n",
+ "dataset = rg.FeedbackDataset.from_argilla(\"customer_assistant\", workspace=\"admin\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If you have skipped the previous steps run this to get the pre-built dataset.\n",
+ "\n",
+ "```python\n",
+ "dataset = rg.FeedbackDataset.from_huggingface(\"argilla/customer_assistant\")\n",
+ "```"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "N5hFLJaHSz9C"
+ },
+ "outputs": [],
+ "source": [
+ "# Adaptation from LlamaIndex's TEXT_QA_PROMPT_TMPL_MSGS[1].content\n",
+ "user_message_prompt = \"\"\"Context information is below.\n",
+ "---------------------\n",
+ "{context_str}\n",
+ "---------------------\n",
+ "Given the context information and not prior knowledge but keeping your Argilla Cloud assistant style, answer the query.\n",
+ "Query: {query_str}\n",
+ "Answer:\n",
+ "\"\"\"\n",
+ "# Adaptation from LlamaIndex's TEXT_QA_SYSTEM_PROMPT\n",
+ "system_prompt = \"\"\"You are an expert customer service assistant for the Argilla Cloud product that is trusted around the world.\n",
+ "Always answer the query using the provided context information, and not prior knowledge.\n",
+ "Some rules to follow:\n",
+ "1. Never directly reference the given context in your answer.\n",
+ "2. Avoid statements like 'Based on the context, ...' or 'The context information ...' or anything along those lines.\n",
+ "\"\"\""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "siPKLnctRLpD"
+ },
+ "outputs": [],
+ "source": [
+ "def formatting_func(\n",
+ " sample: dict,\n",
+ ") -> Union[Tuple[str, str, str, str], List[Tuple[str, str, str, str]]]:\n",
+ " from uuid import uuid4\n",
+ "\n",
+ " if sample[\"response\"]:\n",
+ " chat = str(uuid4())\n",
+ " user_message = user_message_prompt.format(\n",
+ " context_str=sample[\"context\"], query_str=sample[\"user-message\"]\n",
+ " )\n",
+ " return [\n",
+ " (chat, \"0\", \"system\", system_prompt),\n",
+ " (chat, \"1\", \"user\", user_message),\n",
+ " (chat, \"2\", \"assistant\", sample[\"response\"][0][\"value\"]),\n",
+ " ]\n",
+ "\n",
+ "\n",
+ "task = TrainingTask.for_chat_completion(formatting_func=formatting_func)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "SVvdU9EqVWSR"
+ },
+ "source": [
+ "## Fine-tune GPT3.5 with high-quality feedback\n",
+ "\n",
+ "\n",
+ "We fine-tune `gpt-3.5-turbo` with the exported dataset using the Argilla Trainer."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "afLyk289VhBL"
+ },
+ "outputs": [],
+ "source": [
+ "trainer = ArgillaTrainer(\n",
+ " dataset=dataset,\n",
+ " task=task,\n",
+ " framework=\"openai\",\n",
+ ")\n",
+ "trainer.train(output_dir=\"my-ft-openai-model\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "eG-zXJJ2dbi_"
+ },
+ "source": [
+ "## Evaluating base vs fine-tuned with human preference data\n",
+ "\n",
+ "We set up a new feedback dataset for gathering human feedback to evaluate the fine-tuned model against the base model, using the test dataset.\n",
+ "\n",
+ "There are many ways to collect feedback for this phase. The most suitable in this case is human preference data over responses from the two models: *asking our labelers which response is the most accurate and helpful*. We can easily do this with Argilla's `RankingQuestion`.\n",
+ "\n",
+ "Additionally, as both responses can be equally bad, we can ask labelers to write down a correct response. In this case, we would be collecting demonstration data to add to our fine-tuning workflow.\n",
+ "\n",
+ "\n",
+ "### Create dataset and collect feedback\n",
+ "\n",
+ "We set up and publish a new dataset with a `RankingQuestion` and `TextQuestion`, showing our labelers the `user-message` and two responses (from the base and the fine-tuned models)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "tsQT0Ps1igZc"
+ },
+ "outputs": [],
+ "source": [
+ "dataset = rg.FeedbackDataset(\n",
+ " fields=[\n",
+ " rg.TextField(name=\"user-message\"),\n",
+ " rg.TextField(name=\"response-a\"),\n",
+ " rg.TextField(name=\"response-b\"),\n",
+ " ],\n",
+ " questions=[\n",
+ " rg.RankingQuestion(\n",
+ " name=\"preference\",\n",
+ " title=\"Which response is more helpful, harmless, and accurate.\",\n",
+ " values=[\"response-a\", \"response-b\"],\n",
+ " ),\n",
+ " rg.TextQuestion(\n",
+ " name=\"response\",\n",
+ " title=\"If none is good, write a helpful, harmless, accurate response to the user question\",\n",
+ " required=False,\n",
+ " ),\n",
+ " ],\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "95-iBx8fgoGg"
+ },
+ "outputs": [],
+ "source": [
+ "# Read our test questions\n",
+ "questions = load_dataset(\"argilla/cloud_assistant_questions\", split=\"test\")[\"question\"]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "background_save": true
+ },
+ "id": "NPPLoZGae9ia"
+ },
+ "outputs": [],
+ "source": [
+ "# Generate responses with base model\n",
+ "index = VectorStoreIndex.from_documents(documents, service_context=gpt_35_context)\n",
+ "query_engine = index.as_query_engine(similarity_top_k=2)\n",
+ "\n",
+ "contexts = []\n",
+ "base_model_responses = []\n",
+ "\n",
+ "for question in tqdm(questions):\n",
+ " response = query_engine.query(question)\n",
+ " base_model_responses.append(str(response))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "AU9NwKnQgAPu"
+ },
+ "outputs": [],
+ "source": [
+ "# Generate responses with ft model: replace with the id of your ft model\n",
+ "ft_context = ServiceContext.from_defaults(\n",
+ " llm=OpenAI(model=\"ft:gpt-3.5-turbo-...\", temperature=0.3)\n",
+ ")\n",
+ "index = VectorStoreIndex.from_documents(documents, service_context=ft_context)\n",
+ "query_engine = index.as_query_engine(similarity_top_k=2)\n",
+ "\n",
+ "contexts = []\n",
+ "ft_model_responses = []\n",
+ "\n",
+ "for question in tqdm(questions):\n",
+ " response = query_engine.query(question)\n",
+ " ft_model_responses.append(str(response))"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "hsPoYfgJTCOD"
+ },
+ "source": [
+ "An important step here is to randomize the order in which responses are shown.\n",
+ "\n",
+ "If we show the fine-tuned model response always as the first option, we can introduce position bias (labelers always choosing a certain position) or make it evident to users that there are two obviously different models.\n",
+ "\n",
+ "To avoid this, we randomize the position and keep two metadata fields indicating which model has produced `response-a` and `response-b`. When collecting the responses, we'll use this metadata to map the ranking with each model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "0zvH3sDkhMFJ"
+ },
+ "outputs": [],
+ "source": [
+ "records = []\n",
+ "for base, ft, question in zip(base_model_responses, ft_model_responses, questions):\n",
+ " # Randomizing the position is a highly important step to mitigate labeler biases\n",
+ " # Shuffle the order of base and ft\n",
+ " response_a, response_b = random.sample([base, ft], 2)\n",
+ "\n",
+ " # Map the responses back to their model names\n",
+ " models = {base: \"base_model\", ft: \"ft_model\"}\n",
+ " feedback_record = rg.FeedbackRecord(\n",
+ " fields={\n",
+ " \"user-message\": question,\n",
+ " \"response-a\": response_a,\n",
+ " \"response-b\": response_b,\n",
+ " },\n",
+ " metadata={\n",
+ " \"response-a-model\": models[response_a],\n",
+ " \"response-b-model\": models[response_b],\n",
+ " },\n",
+ " )\n",
+ "\n",
+ " records.append(feedback_record)\n",
+ "\n",
+ "dataset = dataset.push_to_argilla(\n",
+ " name=\"finetuned-vs-base-preference\", workspace=\"admin\"\n",
+ ")\n",
+ "dataset.add_records(records)"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now, the dataset is available for collecting feedback with the Argilla UI. Here's a video showing the workflow for labelers:\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "eRiVf39Bblmh"
+ },
+ "source": [
+ "### Retrieve and analyze responses\n",
+ "\n",
+ "We can dynamically collect the responses from our labelers. In this case, we will compute the win rate and ties (as users can indicate both responses are equally good or bad).\n",
+ "\n",
+ "For the tutorial, we only have one user but Argilla Feedback is fully multi-user, which means you can collect feedback from several users for each data point, increasing the quality of the evaluation. \n",
+ "\n",
+ "You can read more about multi-user scenarios and built-in unification methods [on this guide](/practical_guides/collect_responses.md).\n",
+ "\n",
+ "With a very small evaluation set, we can see that the fine-tuned model responses are preferred ~60% of the time, 3x over the base model, and they are both equally good or bad ~20% of the time. \n",
+ "\n",
+ "\n",
+ "Even with a very small fine-tuning and evaluation dataset, this already shows **promising benefits of fine-tuning models for enhancing RAG systems**.\n",
+ "\n",
+ "\n",
+ ""
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "XZKBhlKCbwUV"
+ },
+ "outputs": [],
+ "source": [
+ "# Retrieve the dataset from Argilla\n",
+ "dataset = rg.FeedbackDataset.from_argilla(\n",
+ " name=\"finetuned-vs-base-preference\", workspace=\"admin\"\n",
+ ")\n",
+ "\n",
+ "win_rates = {\"ft_model\": 0, \"base_model\": 0, \"tie\": 0}\n",
+ "\n",
+ "# Compute the win and tie rates\n",
+ "for record in dataset.records:\n",
+ " if len(record.responses) > 0:\n",
+ " for response in record.responses:\n",
+ " model_a = record.metadata[\"response-a-model\"]\n",
+ " model_b = record.metadata[\"response-b-model\"]\n",
+ " preference = response.values[\"preference\"].value\n",
+ " if preference[0].rank > preference[1].rank:\n",
+ " win_rates[model_a] = win_rates[model_a] + 1\n",
+ " elif preference[1].rank > preference[0].rank:\n",
+ " win_rates[model_b] = win_rates[model_b] + 1\n",
+ " else:\n",
+ " win_rates[\"tie\"] = win_rates[\"tie\"] + 1\n",
+ "win_rates\n",
+ "# {'ft_model': 17, 'base_model': 6, 'tie': 5}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "BpoH-OnRnuoL"
+ },
+ "outputs": [],
+ "source": [
+ "# Let's make the labels more explicit\n",
+ "data = {\"gpt3.5-fine-tuned\": 17, \"gpt3.5-base\": 6, \"tie\": 5}\n",
+ "total = sum(data.values())\n",
+ "\n",
+ "# Calculate percentages\n",
+ "percentages = [value / total * 100 for value in data.values()]\n",
+ "\n",
+ "# Settings\n",
+ "colors = [\"blue\", \"grey\", \"black\"]\n",
+ "labels = [f\"{key} ({value:.2f}%)\" for key, value in zip(data.keys(), percentages)]\n",
+ "\n",
+ "# Plotting\n",
+ "plt.figure(figsize=(12, 2))\n",
+ "\n",
+ "# The cumulative percentage is used to shift the starting point of each subsequent segment\n",
+ "cumulative_percentages = 0\n",
+ "\n",
+ "for percent, color, label in zip(percentages, colors, labels):\n",
+ " plt.barh(\"Models\", percent, color=color, label=label, left=cumulative_percentages)\n",
+ " plt.text(\n",
+ " cumulative_percentages + percent / 2,\n",
+ " 0,\n",
+ " label,\n",
+ " ha=\"center\",\n",
+ " va=\"center\",\n",
+ " color=\"white\",\n",
+ " fontsize=10,\n",
+ " )\n",
+ " cumulative_percentages += percent\n",
+ "\n",
+ "plt.gca().axes.get_yaxis().set_visible(False)\n",
+ "plt.xlim(0, 100)\n",
+ "plt.title(\"Model Win Rates\")\n",
+ "plt.legend(loc=\"upper center\", bbox_to_anchor=(0.5, -0.25), ncol=3)\n",
+ "plt.tight_layout()\n",
+ "\n",
+ "# Display\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "hCrnoTw_GIyr"
+ },
+ "source": [
+ "## Appendix: Generating questions with Llama Index\n",
+ "\n",
+ "\n",
+ "We use the `DatasetGenerator` from Llama Index to generate a set of questions using a document about Argilla Cloud."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "kf-hZurHckDf"
+ },
+ "outputs": [],
+ "source": [
+ "UnstructuredReader = download_loader(\"UnstructuredReader\", refresh_cache=True)\n",
+ "loader = UnstructuredReader()\n",
+ "\n",
+ "# You can download this doc from: https://huggingface.co/datasets/argilla/cloud_assistant_questions/raw/main/argilla_cloud.txt\n",
+ "documents = loader.load_data(\"argilla_cloud.txt\")\n",
+ "\n",
+ "gpt_35_context = ServiceContext.from_defaults(\n",
+ " llm=OpenAI(model=\"gpt-3.5-turbo\", temperature=0.4), chunk_size=60\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "wSVCTgEsc1zm"
+ },
+ "outputs": [],
+ "source": [
+ "question_gen_query = (\n",
+ " \"You are customer support and sales expert of Argilla. Your task is to setup \"\n",
+ " \"a set of frequently asked questions about the Argilla Cloud service, offer and plans\"\n",
+ " \"formulate a single question that could be asked by a potential B2B client interested in Argilla Cloud \"\n",
+ " \". Restrict the question to the context information provided and don't ask general questions not related to the service and the context provided.\"\n",
+ ")\n",
+ "\n",
+ "dataset_generator = DatasetGenerator.from_documents(\n",
+ " documents,\n",
+ " question_gen_query=question_gen_query,\n",
+ " service_context=gpt_35_context,\n",
+ " num_questions_per_chunk=100,\n",
+ ")\n",
+ "\n",
+ "questions = dataset_generator.generate_questions_from_nodes(num=300)"
+ ]
+ }
+ ],
+ "metadata": {
+ "colab": {
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
},
- "nbformat": 4,
- "nbformat_minor": 0
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.13"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 0
}
diff --git a/docs/_source/tutorials_and_integrations/tutorials/feedback/fine-tuning-sentencesimilarity-rag.ipynb b/docs/_source/tutorials_and_integrations/tutorials/feedback/fine-tuning-sentencesimilarity-rag.ipynb
index 54b7156b49..2461d17cfb 100644
--- a/docs/_source/tutorials_and_integrations/tutorials/feedback/fine-tuning-sentencesimilarity-rag.ipynb
+++ b/docs/_source/tutorials_and_integrations/tutorials/feedback/fine-tuning-sentencesimilarity-rag.ipynb
@@ -117,7 +117,15 @@
"from tqdm import tqdm\n",
"\n",
"from haystack.document_stores import FAISSDocumentStore\n",
- "from haystack.nodes import PreProcessor, TextConverter, EmbeddingRetriever, PromptNode, PromptTemplate, AnswerParser, SentenceTransformersRanker\n",
+ "from haystack.nodes import (\n",
+ " PreProcessor,\n",
+ " TextConverter,\n",
+ " EmbeddingRetriever,\n",
+ " PromptNode,\n",
+ " PromptTemplate,\n",
+ " AnswerParser,\n",
+ " SentenceTransformersRanker,\n",
+ ")\n",
"from haystack.pipelines import Pipeline\n",
"from haystack.pipelines.standard_pipelines import TextIndexingPipeline"
]
@@ -138,11 +146,7 @@
"# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
"# Replace api_key if you configured a custom API key\n",
"# Replace workspace with the name of your workspace\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"owner.apikey\",\n",
- " workspace=\"admin\"\n",
- ")"
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"owner.apikey\", workspace=\"admin\")"
]
},
{
@@ -165,7 +169,7 @@
"# # Replace api_url with the url to your HF Spaces URL\n",
"# # Replace api_key if you configured a custom API key\n",
"# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
"# api_key=\"admin.apikey\",\n",
"# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
"# )"
@@ -188,9 +192,12 @@
"source": [
"try:\n",
" from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
" tutorial_running()\n",
"except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
]
},
{
@@ -249,7 +256,9 @@
"outputs": [],
"source": [
"# Initialize the DocumentStore\n",
- "document_store = FAISSDocumentStore(faiss_index_factory_str=\"Flat\", similarity=\"dot_product\", embedding_dim=384)\n",
+ "document_store = FAISSDocumentStore(\n",
+ " faiss_index_factory_str=\"Flat\", similarity=\"dot_product\", embedding_dim=384\n",
+ ")\n",
"\n",
"# Initialize the PreProcessor\n",
"preprocessor = PreProcessor(\n",
@@ -297,7 +306,8 @@
"source": [
"# Initialize the EmbeddingRetriever\n",
"retriever = EmbeddingRetriever(\n",
- " document_store=document_store, embedding_model=\"sentence-transformers/multi-qa-mpnet-base-dot-v1\"\n",
+ " document_store=document_store,\n",
+ " embedding_model=\"sentence-transformers/multi-qa-mpnet-base-dot-v1\",\n",
")\n",
"document_store.update_embeddings(retriever)\n",
"\n",
@@ -307,11 +317,13 @@
" Provide a clear and concise response.\n",
" Your answer should be in your own words and be no longer than 50 words.\n",
" \\n\\n Related text: {join(documents, delimiter=new_line, pattern=new_line+'Document[$idx]: $content', str_replace={new_line: ' ', '[': '(', ']': ')'})} \\n Question: {query}; Answer: \"\"\",\n",
- " output_parser=AnswerParser(reference_pattern=r\"Document\\[(\\d+)\\]\"),\n",
+ " output_parser=AnswerParser(reference_pattern=r\"Document\\[(\\d+)\\]\"),\n",
")\n",
"\n",
"# Initialize PromptNode\n",
- "prompt_node = PromptNode(model_name_or_path=\"google/flan-t5-large\", default_prompt_template=rag_prompt)"
+ "prompt_node = PromptNode(\n",
+ " model_name_or_path=\"google/flan-t5-large\", default_prompt_template=rag_prompt\n",
+ ")"
]
},
{
@@ -336,17 +348,16 @@
"questions = dataset[\"train\"][\"question\"]\n",
"answers = []\n",
"contexts = []\n",
- " \n",
+ "\n",
"for question in tqdm(questions):\n",
- " \n",
" # Get the response and save it\n",
" response = pipe.run(query=question)\n",
" answers.append(response[\"answers\"][0].answer)\n",
- " \n",
+ "\n",
" # Get the document contexts and save them\n",
- " prompt = response[\"answers\"][0].meta['prompt']\n",
- " segments = re.split(r'Document\\[\\d+\\]:', prompt)\n",
- " document_segments = [segment.strip() for segment in segments[1:]] \n",
+ " prompt = response[\"answers\"][0].meta[\"prompt\"]\n",
+ " segments = re.split(r\"Document\\[\\d+\\]:\", prompt)\n",
+ " document_segments = [segment.strip() for segment in segments[1:]]\n",
" contexts.append(document_segments)"
]
},
@@ -464,8 +475,13 @@
"# Create the proper records\n",
"records = [\n",
" rg.FeedbackRecord(\n",
- " fields={\"query\": question, \"retrieved_document_1\": context[0], \"retrieved_document_2\": context[1], \"retrieved_document_3\": context[2]},\n",
- " metadata={\"source\": \"flan-t5-large\"}\n",
+ " fields={\n",
+ " \"query\": question,\n",
+ " \"retrieved_document_1\": context[0],\n",
+ " \"retrieved_document_2\": context[1],\n",
+ " \"retrieved_document_3\": context[2],\n",
+ " },\n",
+ " metadata={\"source\": \"flan-t5-large\"},\n",
" )\n",
" for question, context in tqdm(zip(questions, contexts))\n",
"]\n",
@@ -554,7 +570,9 @@
],
"source": [
"# Add the new metadata property\n",
- "metadata = rg.TermsMetadataProperty(name=\"sources\", title=\"Model sources\", values=[\"flan-t5-large/flan-t5-large\"])\n",
+ "metadata = rg.TermsMetadataProperty(\n",
+ " name=\"sources\", title=\"Model sources\", values=[\"flan-t5-large/flan-t5-large\"]\n",
+ ")\n",
"\n",
"dataset_ssim.add_metadata_property(metadata)"
]
@@ -572,7 +590,7 @@
" for j in range(i + 1, len(context)):\n",
" record = rg.FeedbackRecord(\n",
" fields={\"sentence1\": context[i], \"sentence2\": context[j]},\n",
- " metadata={\"sources\": \"flan-t5-large/flan-t5-large\"}\n",
+ " metadata={\"sources\": \"flan-t5-large/flan-t5-large\"},\n",
" )\n",
" records.append(record)\n",
"\n",
@@ -650,11 +668,13 @@
"source": [
"# Define the training task using the formatting function\n",
"def formatting_func(sample):\n",
- "\n",
" records = []\n",
"\n",
" for i in range(1, 4):\n",
- " record = {\"sentence-1\": sample[\"query\"], \"sentence-2\": sample[f\"retrieved_document_{i}\"]}\n",
+ " record = {\n",
+ " \"sentence-1\": sample[\"query\"],\n",
+ " \"sentence-2\": sample[f\"retrieved_document_{i}\"],\n",
+ " }\n",
" values = [resp[\"value\"] for resp in sample[f\"rating_retrieved_document_{i}\"]]\n",
" label = int(values[0])\n",
" record[\"label\"] = label\n",
@@ -662,6 +682,7 @@
"\n",
" return records\n",
"\n",
+ "\n",
"task = TrainingTask.for_sentence_similarity(formatting_func=formatting_func)"
]
},
@@ -676,7 +697,7 @@
" dataset=dataset_rag,\n",
" task=task,\n",
" framework=\"sentence-transformers\",\n",
- " framework_kwargs={\"cross_encoder\": False}\n",
+ " framework_kwargs={\"cross_encoder\": False},\n",
")\n",
"trainer_bi.train(output_dir=\"my_bi_sentence_transformer_model\")"
]
@@ -717,8 +738,11 @@
"source": [
"# Define the training task\n",
"task = TrainingTask.for_sentence_similarity(\n",
- " texts=[dataset_ssim.field_by_name(\"sentence-1\"), dataset_ssim.field_by_name(\"sentence-2\")],\n",
- " label=dataset_ssim.question_by_name(\"similarity\")\n",
+ " texts=[\n",
+ " dataset_ssim.field_by_name(\"sentence-1\"),\n",
+ " dataset_ssim.field_by_name(\"sentence-2\"),\n",
+ " ],\n",
+ " label=dataset_ssim.question_by_name(\"similarity\"),\n",
")"
]
},
@@ -733,7 +757,7 @@
" dataset=dataset_ssim,\n",
" task=task,\n",
" framework=\"sentence-transformers\",\n",
- " framework_kwargs={\"cross_encoder\": True}\n",
+ " framework_kwargs={\"cross_encoder\": True},\n",
")\n",
"trainer_cross.train(output_dir=\"my_cross_sentence_transformer_model\")"
]
@@ -788,7 +812,9 @@
"outputs": [],
"source": [
"# Initialize the SentenceTransformersRanker with out model\n",
- "ranker = SentenceTransformersRanker(model_name_or_path=\"my_cross_sentence_transformer_model\")"
+ "ranker = SentenceTransformersRanker(\n",
+ " model_name_or_path=\"my_cross_sentence_transformer_model\"\n",
+ ")"
]
},
{
@@ -810,11 +836,13 @@
" Provide a clear and concise response.\n",
" Your answer should be in your own words and be no longer than 50 words.\n",
" \\n\\n Related text: {join(documents, delimiter=new_line, pattern=new_line+'Document[$idx]: $content', str_replace={new_line: ' ', '[': '(', ']': ')'})} \\n Question: {query}; Answer: \"\"\",\n",
- " output_parser=AnswerParser(reference_pattern=r\"Document\\[(\\d+)\\]\"),\n",
+ " output_parser=AnswerParser(reference_pattern=r\"Document\\[(\\d+)\\]\"),\n",
")\n",
"\n",
"# Initialize PromptNode\n",
- "prompt_node = PromptNode(model_name_or_path=\"google/flan-t5-large\", default_prompt_template=rag_prompt)"
+ "prompt_node = PromptNode(\n",
+ " model_name_or_path=\"google/flan-t5-large\", default_prompt_template=rag_prompt\n",
+ ")"
]
},
{
@@ -840,17 +868,16 @@
"questions = dataset[\"train\"][\"question\"]\n",
"answers = []\n",
"contexts = []\n",
- " \n",
+ "\n",
"for question in tqdm(questions):\n",
- " \n",
" # Get the response and save it\n",
" response = pipe.run(query=question)\n",
" answers.append(response[\"answers\"][0].answer)\n",
- " \n",
+ "\n",
" # Get the document context and save it\n",
- " prompt = response[\"answers\"][0].meta['prompt']\n",
- " segments = re.split(r'Document\\[\\d+\\]:', prompt)\n",
- " document_segments = [segment.strip() for segment in segments[1:]] \n",
+ " prompt = response[\"answers\"][0].meta[\"prompt\"]\n",
+ " segments = re.split(r\"Document\\[\\d+\\]:\", prompt)\n",
+ " document_segments = [segment.strip() for segment in segments[1:]]\n",
" contexts.append(document_segments)"
]
},
diff --git a/docs/_source/tutorials_and_integrations/tutorials/feedback/labelling-feedback-langchain-syntethic.ipynb b/docs/_source/tutorials_and_integrations/tutorials/feedback/labelling-feedback-langchain-syntethic.ipynb
index ba95bdafc8..2b1061fe7d 100644
--- a/docs/_source/tutorials_and_integrations/tutorials/feedback/labelling-feedback-langchain-syntethic.ipynb
+++ b/docs/_source/tutorials_and_integrations/tutorials/feedback/labelling-feedback-langchain-syntethic.ipynb
@@ -1,2874 +1,2895 @@
{
- "cells": [
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# 🎡 Create synthetic data and annotations with LLMs\n",
+ "\n",
+ "LLMs are diverse and can be used for many different tasks. Besides cool chat interactions, LLMs can be powerful tools for creating synthetic data and providing initial suggestions for labelling tasks for which you don't have any data yet. This way anyone can easily get a head start on bootstrapping a project.\n",
+ "\n",
+ "In this example, we will demonstrate how to use different LLM tools, like `openai`, `transformers`, `langchain` and `outlines`, to create synthetic data and we can leverage those same LLMs for providing initial annotation or suggestions.\n",
+ "\n",
+ "If you want a more basic introduction to synthetic data with our `ArgillaCallbackHandler` for `langchain`, you can take a look at [this practical guide](../../../tutorials_and_integrations/integrations/use_argilla_callback_in_langchain.md).\n",
+ "\n",
+ "
\n",
+ "\n",
+ "Warning\n",
+ "\n",
+ "Do keep in mind that LLMs have licenses and not every LLM can be used for creating synthetic data in every operational setting. Please check the license of the LLM you are using before using it for creating synthetic data.\n",
+ "\n",
+ "
\n",
+ "\n",
+ "Let's get started!\n",
+ "\n",
+ "\n",
+ "\n",
+ "
\n",
+ "\n",
+ "Note \n",
+ "\n",
+ "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
+ "\n",
+ "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
+ "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
+ "\n",
+ "
\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Setup\n",
+ "\n",
+ "For this tutorial, you will need to have an Argilla server running. If you don't have one already, check out our [Quickstart](../../../getting_started/quickstart.md) or [Installation](../../../getting_started/quickstart_installation.ipynb) pages. Once you do, complete the following steps:\n",
+ "\n",
+ "1. Install the Argilla client and the required third-party libraries using `pip`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "yN2atS0RE2pF"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install argilla openai langchain outlines tiktoken transformers ipywidgets jupyter"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "2. Let's make the necessary imports:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {
+ "id": "POQgkfrWEg1u"
+ },
+ "outputs": [],
+ "source": [
+ "import argilla as rg\n",
+ "\n",
+ "import os\n",
+ "import random\n",
+ "\n",
+ "from langchain import OpenAI, PromptTemplate, LLMChain\n",
+ "from langchain.output_parsers import CommaSeparatedListOutputParser\n",
+ "from outlines import models, text\n",
+ "from outlines.text import generate"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "3. If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
+ "# Replace api_key if you configured a custom API key\n",
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"admin.apikey\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Set the HF_TOKEN environment variable\n",
+ "# import os\n",
+ "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
+ "\n",
+ "# # Replace api_url with the url to your HF Spaces URL\n",
+ "# # Replace api_key if you configured a custom API key\n",
+ "# rg.init(\n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
+ "# api_key=\"admin.apikey\",\n",
+ "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
+ "# )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "4. We also need to set your OpenAI API credentials by [creating an API key](https://platform.openai.com/docs/quickstart/add-your-api-key) and setting defining the `OPENAI_API_KEY` environment variable."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Enable Telemetry\n",
+ "\n",
+ "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
+ " tutorial_running()\n",
+ "except ImportError:\n",
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Define a FeedbackDataset"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "In this example, we will create a synthetic dataset for a banking customer care scenario. We assume that customers will write `text` requests. These requests should then be classified for `sentiment` and `topics`. The `topics` will be a multi-label classification and can be used to route the request to the correct department. The `sentiment` will be used using a single-label classification to determine if the request needs to be handled with priority."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "metadata": {
+ "id": "0UsoG5OtE11w"
+ },
+ "outputs": [],
+ "source": [
+ "sentiment = [\"positive\", \"neutral\", \"negative\"]\n",
+ "topic = [\"new_card\", \"mortgage\", \"application\", \"payments\"]\n",
+ "\n",
+ "dataset = rg.FeedbackDataset(\n",
+ " fields=[rg.TextField(name=\"text\")],\n",
+ " questions=[\n",
+ " rg.LabelQuestion(\n",
+ " name=\"sentiment\",\n",
+ " title=\"What is the sentiment of the message?\",\n",
+ " labels=sentiment,\n",
+ " ),\n",
+ " rg.MultiLabelQuestion(\n",
+ " name=\"topics\",\n",
+ " title=\"Select the topic(s) of the message?\",\n",
+ " labels=topic,\n",
+ " visible_labels=4,\n",
+ " ),\n",
+ " ],\n",
+ " guidelines=(\n",
+ " \"This dataset contains messages from a bank's customer support chatbot. \"\n",
+ " \"The goal is to label the sentiment and topics of the messages.\"\n",
+ " ),\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Create synthetic data"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We will use LLMs to generate the synthetic data for each step of the NLP task. First, we will create `text` requests from customers for a bank. Next, we will create input for the `LabelQuestion` to assess the sentiment of the requests and lastly, we will create input for the `MultiLabelQuestion` to classify the requests.\n",
+ "\n",
+ "We will do this using [OpenAI](https://github.com/openai/openai-python) models in combination with the [LangChain](https://github.com/langchain-ai/langchain) and open-source transformer-based models in combination with [Outlines](https://github.com/normal-computing/outlines) packages.\n",
+ "\n",
+ " | The LangChain framework is a wrapper around LLM models that allows for easier data-aware and agent-based LLM models. \n",
+ "\n",
+ " | Outlines is a Python library to write reliable programs for conditional generation during interactions with generative models.\n",
+ " \n",
+ "
\n",
+ "\n",
+ "Note \n",
+ "\n",
+ "The process of prompt engineering is a trial-and-error process. Changes somewhere might result in undesirable effects in another place in the language chain. The examples below are just a starting point and can be improved by experimenting with different prompts and templates.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Initialize Generative model\n",
+ "\n",
+ "#### LangChain with OpenAI\n",
+ "\n",
+ "For the usage of LangChain you need to pass the `OPENAI_API_KEY` environment variable to the `OpenAI` class. You can do this by using the `os` package. The `model` variable is then ready to use in the examples below."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "os.environ[\"OPENAI_API_KEY\"] = \"sk-...\"\n",
+ "openai_model = OpenAI(api_key=os.environ.get(\"OPENAI_API_KEY\"))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Outlines with Transformers\n",
+ "\n",
+ "Even though Outlines does provide some support for OpenAI, we will use basic transformers for this example. You can use any generative model from the [HuggingFace model hub](https://huggingface.co/models?pipeline_tag=text-generation&sort=trending) by passing the name of the model to the `transformers` function. The `model` variable is then ready to use in the examples below."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "transformer_model = models.transformers(\"gpt2\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### `TextField`\n",
+ "\n",
+ "For creating a review, we rely on free text generation based on a prompt. This should be good enough for our purposes of creating a synthetic dataset as well as keeping the process as simple as possible."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### LangChain\n",
+ "\n",
+ "OpenAI models have been instruction-tuned and can thus be used via LangChain to generate synthetic data. This is done using a `PrompTemplate` that infers information from `topic` and `sentiment` variables that are passed to the `predict()` method."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [
{
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# 🎡 Create synthetic data and annotations with LLMs\n",
- "\n",
- "LLMs are diverse and can be used for many different tasks. Besides cool chat interactions, LLMs can be powerful tools for creating synthetic data and providing initial suggestions for labelling tasks for which you don't have any data yet. This way anyone can easily get a head start on bootstrapping a project.\n",
- "\n",
- "In this example, we will demonstrate how to use different LLM tools, like `openai`, `transformers`, `langchain` and `outlines`, to create synthetic data and we can leverage those same LLMs for providing initial annotation or suggestions.\n",
- "\n",
- "If you want a more basic introduction to synthetic data with our `ArgillaCallbackHandler` for `langchain`, you can take a look at [this practical guide](../../../tutorials_and_integrations/integrations/use_argilla_callback_in_langchain.md).\n",
- "\n",
- "
\n",
- "\n",
- "Warning\n",
- "\n",
- "Do keep in mind that LLMs have licenses and not every LLM can be used for creating synthetic data in every operational setting. Please check the license of the LLM you are using before using it for creating synthetic data.\n",
- "\n",
- "
\n",
- "\n",
- "Let's get started!\n",
- "\n",
- "\n",
- "\n",
- "
\n",
- "\n",
- "Note \n",
- "\n",
- "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
- "\n",
- "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
- "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
- "\n",
- "
\n"
+ "data": {
+ "text/plain": [
+ "'I recently had the pleasure of working with the mortgage team at this bank, and I can confidently say that their level of service and expertise was second to none. They answered all of my questions quickly and took the time to explain the process to me in detail. I felt like they genuinely had my best interests at heart and they made the process of obtaining a mortgage as smooth and stress-free as possible. I would highly recommend this bank for anyone looking to take out a mortgage.'"
]
- },
+ },
+ "execution_count": 35,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "template = (\n",
+ " \"Write a customer review for a bank. \"\n",
+ " \"Do that for topic of {topic}. \"\n",
+ " \"Do that with one a {sentiment} sentiment.\"\n",
+ ")\n",
+ "prompt = PromptTemplate(template=template, input_variables=[\"topic\", \"sentiment\"])\n",
+ "llm_chain_review = LLMChain(prompt=prompt, llm=openai_model)\n",
+ "\n",
+ "\n",
+ "def generate_langchain_review():\n",
+ " return llm_chain_review.predict(\n",
+ " topic=random.choice(topic), sentiment=random.choice(sentiment)\n",
+ " ).strip()\n",
+ "\n",
+ "\n",
+ "generate_langchain_review()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We will now create a function that can generate `n`-random examples to evaluate the performance. As we can expect from the recent generation of OpenAI models, the results look good and seem diverse enough to be used as synthetic data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Setup\n",
- "\n",
- "For this tutorial, you will need to have an Argilla server running. If you don't have one already, check out our [Quickstart](../../../getting_started/quickstart.md) or [Installation](../../../getting_started/quickstart_installation.ipynb) pages. Once you do, complete the following steps:\n",
- "\n",
- "1. Install the Argilla client and the required third-party libraries using `pip`:"
+ "data": {
+ "text/plain": [
+ "[\"I've been a customer of this bank for over 5 years, and I've been completely satisfied with their payment services. The online payment system is easy to use and the customer service team is always quick to respond to any questions I have. I never have to worry about my payments being delayed or lost, which is always reassuring. Highly recommend this bank for anyone looking for reliable payment services!\",\n",
+ " \"I recently secured a mortgage with this bank and was so impressed with the level of service I received. From the start, the staff was friendly, knowledgeable, and willing to go above and beyond to get me the best deal. The process was straightforward and the paperwork was easy to understand. I'm thrilled with my new mortgage and would highly recommend this bank to anyone looking for a mortgage.\"]"
]
- },
+ },
+ "execution_count": 38,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "def generate_n_langchain_reviews(n=2):\n",
+ " reviews = []\n",
+ " for n in range(n):\n",
+ " reviews.append(generate_langchain_review())\n",
+ " return reviews\n",
+ "\n",
+ "\n",
+ "langchain_reviews = generate_n_langchain_reviews()\n",
+ "langchain_reviews"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Outlines\n",
+ "\n",
+ "Not all generative models are instruction tuned and as useful as modern-day LLMs. So take into account that this should be reflected in your `prompt` and the expected quality of the generated text. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "yN2atS0RE2pF"
- },
- "outputs": [],
- "source": [
- "!pip install argilla openai langchain outlines tiktoken transformers ipywidgets jupyter"
+ "data": {
+ "text/plain": [
+ "\"Because of technical questions and consumer protection. Telephone orders are not placed in the bank's database because where homeowners need to know that their bank is registered here, this protection providing a system to check their record is not protected by even their own state laws, which is why I don't believe criminal laws should be used to enforce the bank ATM login, nor should neutral other town or guild banking providers be regulated. These accountants have followed the local law explanations, and they do not deserve criminal sanction for allowing a\""
]
- },
+ },
+ "execution_count": 35,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "@text.prompt\n",
+ "def generator(topic, sentiment):\n",
+ " \"\"\"\n",
+ " The customer service for {{ topic }} of the bank is {{ sentiment }} because\n",
+ " \"\"\"\n",
+ "\n",
+ "\n",
+ "def generate_outlines_review():\n",
+ " prompt = generator(random.choice(topic), random.choice(sentiment))\n",
+ " answer = text.generate.continuation(transformer_model, max_tokens=100)(prompt)\n",
+ " answer = \"Because\" + answer\n",
+ " return answer\n",
+ "\n",
+ "\n",
+ "generate_outlines_review()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We will now create a function that can generate `n`-random examples. Looking at the examples, the model seems to generate roughly related texts but in general, the quality proves poorer. It can therefore be recommended to use another type of models, which might be instruction tuned to ensure a higher quality generation. Additionally, Outlines offers more dynamic control over the generation process, which might be used to improve the quality of the generated text too."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "2. Let's make the necessary imports:"
+ "data": {
+ "text/plain": [
+ "['Because of damaged card and adds some other attachments from other data on the ToS or database file.\"',\n",
+ " 'Because of jurisdictional issues,\" said the FTC\\'s executive director, Paul R. Matthewi. \"Technology seems to be without limits in the fraud marketplace as we moved toward a new way of remote law enforcement and convenience.\"\\n\\nIt is unclear, however, how people will get paid—or how many will be affected. In TekSavvy, which relies on similar technology many consumer goods companies rely on to keep their customers healthy, some of America\\'s top credit card holders appear to still need money,']"
]
- },
+ },
+ "execution_count": 53,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "def generate_n_outlines_reviews(n=2):\n",
+ " reviews = []\n",
+ " for n in range(n):\n",
+ " reviews.append(generate_outlines_review())\n",
+ " return reviews\n",
+ "\n",
+ "\n",
+ "outlines_reviews = generate_n_outlines_reviews()\n",
+ "outlines_reviews"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### `LabelQuestion`"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "For this step, we will re-use the generated reviews from `langchain_reviews` and `outlines_reviews` and label their sentiment using the respective frameworks. This will be done by assuming a `str` to be returned from both of the lists of `sentiment` defined above."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### LangChain\n",
+ "\n",
+ "We are using a jinja-like `template`, which requires us to define the basic `prompt` as an `input_variable` for LangChain. For the initial example, we are using a demo."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 24,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": 13,
- "metadata": {
- "id": "POQgkfrWEg1u"
- },
- "outputs": [],
- "source": [
- "import argilla as rg\n",
- "\n",
- "import os\n",
- "import random\n",
- "\n",
- "from langchain import OpenAI, PromptTemplate, LLMChain\n",
- "from langchain.output_parsers import CommaSeparatedListOutputParser\n",
- "from outlines import models, text\n",
- "from outlines.text import generate"
+ "data": {
+ "text/plain": [
+ "'positive'"
]
- },
+ },
+ "execution_count": 24,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "template = (\n",
+ " f\"Choose from the sentiments: {sentiment}. Return a single sentiment.{{prompt}}\"\n",
+ ")\n",
+ "prompt = PromptTemplate(template=template, input_variables=[\"prompt\"])\n",
+ "llm_chain_sentiment = LLMChain(prompt=prompt, llm=openai_model)\n",
+ "\n",
+ "\n",
+ "def get_sentiment_from_langchain(text: str) -> str:\n",
+ " return llm_chain_sentiment.predict(prompt=text).strip().lower()\n",
+ "\n",
+ "\n",
+ "get_sentiment_from_langchain(\"I love langchain and openai for sentiment labelling.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We get the sentiment labels for the generated `langchain_reviews`. We can see that the sentiment labels are not always correct, but they are mostly correct. This is because the LLMs are not perfect, but they are good enough to be used for synthetic data generation and providing suggestions for human annotators."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 46,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "3. If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
+ "data": {
+ "text/plain": [
+ "['positive', 'positive']"
]
- },
+ },
+ "execution_count": 46,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "langchain_sentiment = [\n",
+ " get_sentiment_from_langchain(reviews) for reviews in langchain_reviews\n",
+ "]\n",
+ "langchain_sentiment"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Outlines\n",
+ "\n",
+ "Outlines provide an out-of-the-box implementation for guided labeling with generative, however, in some cases [(zero-shot) classification models from the HuggingFace library](https://huggingface.co/models?pipeline_tag=text-classification&sort=trending) can be used to provide a good point for the providing suggestions during a labeling process too. Take a look at our [example with SetFit](labelling-feedback-setfit.ipynb)."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 51,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
- "# Replace api_key if you configured a custom API key\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"admin.apikey\"\n",
- ")"
+ "data": {
+ "text/plain": [
+ "'positive'"
]
- },
+ },
+ "execution_count": 51,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "def get_sentiment_from_outlines(text: Union[str, list]) -> str:\n",
+ " return generate.choice(transformer_model, sentiment)(text)\n",
+ "\n",
+ "\n",
+ "get_sentiment_from_outlines(\"I love outlines and transformers for sentiment labelling.\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We can use the `choice`-methods with a list of strings too."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ "data": {
+ "text/plain": [
+ "['neutral', 'positive']"
]
- },
+ },
+ "execution_count": 55,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "outlines_sentiment = get_sentiment_from_outlines(outlines_reviews)\n",
+ "outlines_sentiment"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### `MultiLabelQuestion`\n",
+ "\n",
+ "For this step, we will re-use the generated reviews from `lanchain_reviews` and `outlines_reviews` and label their topics as part of a multi-label classification problem."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Langchain\n",
+ "\n",
+ "Note that we are now using an output parser as a post-processing step for the returned output. We do this to ensure that we can obtain a `List[str]`. We will use the built-in `CommaSeparatedListOutputParser`, which split strings by comma and returns a list of strings as output. And we are using the jinja-like templating in a similar way as with the `SingleLabelQuestion`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 30,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# # Set the HF_TOKEN environment variable\n",
- "# import os\n",
- "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
- "\n",
- "# # Replace api_url with the url to your HF Spaces URL\n",
- "# # Replace api_key if you configured a custom API key\n",
- "# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
- "# api_key=\"admin.apikey\",\n",
- "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
- "# )"
+ "data": {
+ "text/plain": [
+ "['new_card', 'mortgage', 'application', 'payments']"
]
- },
+ },
+ "execution_count": 30,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "output_parser = CommaSeparatedListOutputParser()\n",
+ "template = (\n",
+ " f\"Classify the text as the following topics: {topic}. \"\n",
+ " \"Return zero or more topics as a comma-separated list. If zero return an empty string. \"\n",
+ " \"{prompt}\"\n",
+ ")\n",
+ "prompt = PromptTemplate(\n",
+ " template=template, input_variables=[\"prompt\"], output_parser=output_parser\n",
+ ")\n",
+ "llm_chain_topics = LLMChain(\n",
+ " prompt=prompt, llm=openai_model, output_parser=output_parser\n",
+ ")\n",
+ "\n",
+ "\n",
+ "def get_topics_from_langchain(text: str) -> str:\n",
+ " return [\n",
+ " topic.lower() for topic in llm_chain_topics.predict(prompt=text) if topic != \"\"\n",
+ " ]\n",
+ "\n",
+ "\n",
+ "get_topics_from_langchain(\n",
+ " f\"I love extracting {topic} with and openai and langchain for topic labelling.\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We get the topic labels for the generated `langchain_reviews`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 67,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "4. We also need to set your OpenAI API credentials by [creating an API key](https://platform.openai.com/docs/quickstart/add-your-api-key) and setting defining the `OPENAI_API_KEY` environment variable."
+ "data": {
+ "text/plain": [
+ "[['new_card'], ['mortgage', 'application']]"
]
- },
+ },
+ "execution_count": 67,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "langchain_topics = [get_topics_from_langchain(review) for review in langchain_reviews]\n",
+ "langchain_topics"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "#### Outlines\n",
+ "\n",
+ "Outlines does not have a direct way to generate data from choices but we are able to leverage their `Pydantic` integration to generate a `json` schema. Note that this is a hacky way to facilitate this guided generation and is not officially mentioned in [the paper behind the outlines package](https://arxiv.org/pdf/2307.09702.pdf). \n",
+ "\n",
+ "Additionally, the use of `json` requires `pydantic>=2`.\n",
+ "\n",
+ "```python\n",
+ "# DEMO CODE\n",
+ "class Topic(BaseModel):\n",
+ " new_card: bool = False\n",
+ " mortgage: bool = False\n",
+ " application: bool = False\n",
+ " payments: bool = False\n",
+ "\n",
+ "def get_topics_from_outlines(text: str) -> str:\n",
+ " topics = []\n",
+ " json_data = generate.json(transformer_model, Topic)(langchain_reviews[0])\n",
+ " for key, value in json_data.items():\n",
+ " if value:\n",
+ " topics.append(key)\n",
+ " return topics\n",
+ "\n",
+ "get_topics_from_outlines(f\"I love extracting {topic} with and outlines and transformers for topic labelling.\")\n",
+ "```"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Create synthetic records"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now we have our synthetic data and predictions, we can use them to create Argilla records. We will create completely artificial records from the `text` for the `TextField` and we will assign the `sentiment` and `topics` as model suggestions for the `LabelQuestion` and `MultiLabelQuestion`, respectively. These suggestions will help the annotators to label the data faster and more accurately, but instead of using them as suggestions, you would also be able to apply them as [annotated responses](../../../practical_guides/create_update_dataset/suggestions_and_responses.md) directly."
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "For demo purposes, we will only create records with synthetic data obtained from `langchain`."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 31,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Enable Telemetry\n",
- "\n",
- "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
+ "data": {
+ "text/plain": [
+ "FeedbackRecord(fields={'text': '\\n\\nI recently applied for a mortgage at this bank and the process was simple and straightforward. The customer service team was helpful and knowledgeable, and their rates were competitive. Overall, I had a good experience.'}, metadata={}, responses=[], suggestions=(SuggestionSchema(question_id=None, question_name='sentiment', type=None, score=None, value='positive', agent=None), SuggestionSchema(question_id=None, question_name='topics', type=None, score=None, value=['mortgage', 'application'], agent=None)), external_id=None)"
]
+ },
+ "execution_count": 31,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "def create_synthetic_record():\n",
+ " review = generate_langchain_review()\n",
+ " record = rg.FeedbackRecord(\n",
+ " fields={\n",
+ " \"text\": review,\n",
+ " }\n",
+ " )\n",
+ " sentiment = get_sentiment_from_langchain(review)\n",
+ " topics = get_topics_from_langchain(review)\n",
+ " record.update(\n",
+ " suggestions=[\n",
+ " {\"question_name\": \"sentiment\", \"value\": sentiment},\n",
+ " {\"question_name\": \"topics\", \"value\": topics},\n",
+ " ]\n",
+ " )\n",
+ " return record\n",
+ "\n",
+ "\n",
+ "record = create_synthetic_record()\n",
+ "record"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We will then add the synthetic `record` to the dataset, and upload the model data and dataset to the Argilla server"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "dataset.add_records([record])\n",
+ "remote_dataset = dataset.push_to_argilla(name=\"synthetic-data\", workspace=\"argilla\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Conclusion\n",
+ "\n",
+ "In this tutorial, we have covered how to create synthetic data using OpenAI and Lanchain, or Transformers and Outlines. We have highlighted some caveats to synthetic data generation when it comes to prompt engineering. Finally, we've shown how to use this synthesized data as input and suggestions for Argilla records.\n",
+ "\n",
+ "To learn more about LLMs, LangChain and OpenAI check out these links:\n",
+ "\n",
+ "- [Outlines](https://github.com/normal-computing/outlines)\n",
+ "- [LangChain](https://github.com/langchain-ai/langchain)"
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "gpuType": "T4",
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "argilla",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.6"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "2d98cb9bf90a932b5bf8e86e91214497eb0e38eb318595fbd6fbd5460fe92036"
+ }
+ },
+ "widgets": {
+ "application/vnd.jupyter.widget-state+json": {
+ "04150cf7e9a74a04aafa94d394553630": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "try:\n",
- " from argilla.utils.telemetry import tutorial_running\n",
- " tutorial_running()\n",
- "except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
- ]
+ "0447a98b5dfe42c899273b9c37bdadad": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Define a FeedbackDataset"
- ]
+ "0c010df989eb497c810a6f960c6ea41b": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "In this example, we will create a synthetic dataset for a banking customer care scenario. We assume that customers will write `text` requests. These requests should then be classified for `sentiment` and `topics`. The `topics` will be a multi-label classification and can be used to route the request to the correct department. The `sentiment` will be used using a single-label classification to determine if the request needs to be handled with priority."
- ]
+ "0d7acd8e1a394336aa146e2a442f672c": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
},
- {
- "cell_type": "code",
- "execution_count": 33,
- "metadata": {
- "id": "0UsoG5OtE11w"
- },
- "outputs": [],
- "source": [
- "sentiment = [\"positive\", \"neutral\", \"negative\"]\n",
- "topic = [\"new_card\", \"mortgage\", \"application\", \"payments\"]\n",
- "\n",
- "dataset = rg.FeedbackDataset(\n",
- " fields = [rg.TextField(name=\"text\")],\n",
- " questions = [\n",
- " rg.LabelQuestion(\n",
- " name=\"sentiment\",\n",
- " title=\"What is the sentiment of the message?\",\n",
- " labels=sentiment\n",
- " ),\n",
- " rg.MultiLabelQuestion(\n",
- " name=\"topics\",\n",
- " title=\"Select the topic(s) of the message?\",\n",
- " labels=topic,\n",
- " visible_labels=4\n",
- " )\n",
- " ],\n",
- " guidelines=(\n",
- " \"This dataset contains messages from a bank's customer support chatbot. \"\n",
- " \"The goal is to label the sentiment and topics of the messages.\"\n",
- " )\n",
- ")"
- ]
+ "16993356757e4ee5b7f8042d58c96e17": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Create synthetic data"
- ]
+ "16d42bc00dfe4467a1da86b1d2391d0d": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "We will use LLMs to generate the synthetic data for each step of the NLP task. First, we will create `text` requests from customers for a bank. Next, we will create input for the `LabelQuestion` to assess the sentiment of the requests and lastly, we will create input for the `MultiLabelQuestion` to classify the requests.\n",
- "\n",
- "We will do this using [OpenAI](https://github.com/openai/openai-python) models in combination with the [LangChain](https://github.com/langchain-ai/langchain) and open-source transformer-based models in combination with [Outlines](https://github.com/normal-computing/outlines) packages.\n",
- "\n",
- " | The LangChain framework is a wrapper around LLM models that allows for easier data-aware and agent-based LLM models. \n",
- "\n",
- " | Outlines is a Python library to write reliable programs for conditional generation during interactions with generative models.\n",
- " \n",
- "
\n",
- "\n",
- "Note \n",
- "\n",
- "The process of prompt engineering is a trial-and-error process. Changes somewhere might result in undesirable effects in another place in the language chain. The examples below are just a starting point and can be improved by experimenting with different prompts and templates.\n",
- "\n",
- "
\n",
- "\n",
- "Note \n",
- "\n",
- "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
- "\n",
- "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
- "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
- "\n",
- "
\n"
- ]
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# ✨ Add zero-shot text classification suggestions using SetFit\n",
+ "\n",
+ "Suggestions are a wonderful way to make things easier and faster for your annotation team. These preselected options will make the labelling process more efficient, as they will only need to correct the suggestions. \n",
+ "\n",
+ "In this example, we will demonstrate how to implement a zero-shot approach using SetFit to get some initial suggestions for dataset that combines two text classification tasks that include a `LabelQuestion` and a `MultiLabelQuestion`.\n",
+ "\n",
+ "Let's get started!\n",
+ "\n",
+ "\n",
+ "\n",
+ "
\n",
+ "\n",
+ "Note \n",
+ "\n",
+ "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
+ "\n",
+ "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
+ "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
+ "\n",
+ "
\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Setup\n",
+ "\n",
+ "For this tutorial, you will need to have an Argilla server running. If you don't have one already, check out our [Quickstart](../../../getting_started/quickstart.md) or [Installation](../../../getting_started/quickstart_installation.ipynb) pages. Once you do, complete the following steps:\n",
+ "\n",
+ "1. Install the Argilla client and the required third-party libraries using `pip`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "yN2atS0RE2pF"
+ },
+ "outputs": [],
+ "source": [
+ "!pip install argilla setfit"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "2. Let's make the necessary imports:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "POQgkfrWEg1u"
+ },
+ "outputs": [],
+ "source": [
+ "import argilla as rg\n",
+ "from datasets import load_dataset\n",
+ "from setfit import get_templated_dataset\n",
+ "from setfit import SetFitModel, SetFitTrainer"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "3. If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
+ "# Replace api_key if you configured a custom API key\n",
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"admin.apikey\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Set the HF_TOKEN environment variable\n",
+ "# import os\n",
+ "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
+ "\n",
+ "# # Replace api_url with the url to your HF Spaces URL\n",
+ "# # Replace api_key if you configured a custom API key\n",
+ "# rg.init(\n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
+ "# api_key=\"admin.apikey\",\n",
+ "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
+ "# )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Enable Telemetry\n",
+ "\n",
+ "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
+ " tutorial_running()\n",
+ "except ImportError:\n",
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Configure the dataset"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "In this example, we will load a popular open-source dataset that has customer requests in the banking domain."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "0UsoG5OtE11w"
+ },
+ "outputs": [],
+ "source": [
+ "data = load_dataset(\"PolyAI/banking77\", split=\"test\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "We will configure our dataset with two different questions so that we can work with two text classification tasks at the same time. In this case, we will load the original labels of this dataset to make a multi-label classification of the topics mentioned in the request and we will also set up a question to classify the sentiment of the request as either \"positive\", \"neutral\" or \"negative\"."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "KKu2QplpFDgw"
+ },
+ "outputs": [],
+ "source": [
+ "dataset = rg.FeedbackDataset(\n",
+ " fields=[rg.TextField(name=\"text\")],\n",
+ " questions=[\n",
+ " rg.MultiLabelQuestion(\n",
+ " name=\"topics\",\n",
+ " title=\"Select the topic(s) of the request\",\n",
+ " labels=data.info.features[\n",
+ " \"label\"\n",
+ " ].names, # these are the original labels present in the dataset\n",
+ " visible_labels=10,\n",
+ " ),\n",
+ " rg.LabelQuestion(\n",
+ " name=\"sentiment\",\n",
+ " title=\"What is the sentiment of the message?\",\n",
+ " labels=[\"positive\", \"neutral\", \"negative\"],\n",
+ " ),\n",
+ " ],\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Train the models"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now we will use the data we loaded and the labels and questions we configured for our dataset to train a zero-shot text classification model for each of the questions in our dataset."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def train_model(question_name, template, multi_label=False):\n",
+ " # build a training dataset that uses the labels of a specific question in our Argilla dataset\n",
+ " train_dataset = get_templated_dataset(\n",
+ " candidate_labels=dataset.question_by_name(question_name).labels,\n",
+ " sample_size=8,\n",
+ " template=template,\n",
+ " multi_label=multi_label,\n",
+ " )\n",
+ "\n",
+ " # train a model using the training dataset we just built\n",
+ " if multi_label:\n",
+ " model = SetFitModel.from_pretrained(\n",
+ " \"all-MiniLM-L6-v2\", multi_target_strategy=\"one-vs-rest\"\n",
+ " )\n",
+ " else:\n",
+ " model = SetFitModel.from_pretrained(\"all-MiniLM-L6-v2\")\n",
+ "\n",
+ " trainer = SetFitTrainer(model=model, train_dataset=train_dataset)\n",
+ " trainer.train()\n",
+ " return model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 276,
+ "referenced_widgets": [
+ "503d373bd18b4b79a1f694916734d903",
+ "6e9e5e1ac58945d0926a85c1fd29ab17",
+ "cc9ccdfefca941e1813258a19afe64ed",
+ "c2238acd18b844c0bb517d670b76ca5c",
+ "90eec4e8ae8b42268548588db2fcbf49",
+ "501d213a24064f998d4d3c45255d02b7",
+ "3d282336f5c3425386a417866f367007",
+ "7b96b0a21eba4ad5a4c12534940b3591",
+ "571fd48c2da8432e8a74e7b318eb6042",
+ "1d58b40ad6a54c25bd451eda4e7d8069",
+ "5e0377b4b48c441a8d747ea904c3207b",
+ "38bfdddef0444c0baf9d29248689f846",
+ "3f5aed26eeef4182b360085d83ae795d",
+ "255d62fb39454098ab3701753d8d67d6",
+ "25f9bca647f44645b85a644f03807095",
+ "ae7fc579502e46f7861e402580586b28",
+ "6143886f7acc4591ae5f79ce6f67af4a",
+ "486c1a817552432c8fb20e59d0a3f079",
+ "77bd2b1f5e57441ab729c6e517279834",
+ "bc0c58d9d798437fb1d40277d8777777",
+ "fa5df54e161e40dbbb21ed96c879444e",
+ "16993356757e4ee5b7f8042d58c96e17",
+ "d11aa6a0c8c54481b6cc2c80d1fa0ba1",
+ "a9ce0af78a2241e697a22229db7840ab",
+ "ae6ffc6572b54c059196983da4ff2d79",
+ "980f36d72cfa403aad67e871aecba890",
+ "5692de58835a466695fcc8f0d5976b74",
+ "7a12fbf5400a468fbdce4b2b2008eefc",
+ "04150cf7e9a74a04aafa94d394553630",
+ "9a7c8861a37b41eba191059546f5dd5d",
+ "217760080e494d2d9b0582910d121a28",
+ "f5e35991e6d849eca73282c9c359000a",
+ "5a06b8d12b494daeb0624f2e39e06e67"
+ ]
},
+ "id": "U9TVO355a2np",
+ "outputId": "7d6b6b60-6f49-4308-a2e6-ac24bf99bf72"
+ },
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Setup\n",
- "\n",
- "For this tutorial, you will need to have an Argilla server running. If you don't have one already, check out our [Quickstart](../../../getting_started/quickstart.md) or [Installation](../../../getting_started/quickstart_installation.ipynb) pages. Once you do, complete the following steps:\n",
- "\n",
- "1. Install the Argilla client and the required third-party libraries using `pip`:"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "config.json not found in HuggingFace Hub.\n",
+ "WARNING:huggingface_hub.hub_mixin:config.json not found in HuggingFace Hub.\n",
+ "model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference.\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "yN2atS0RE2pF"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "503d373bd18b4b79a1f694916734d903",
+ "version_major": 2,
+ "version_minor": 0
},
- "outputs": [],
- "source": [
- "!pip install argilla setfit"
+ "text/plain": [
+ "Generating Training Pairs: 0%| | 0/20 [00:00, ?it/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "2. Let's make the necessary imports:"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "***** Running training *****\n",
+ " Num examples = 24640\n",
+ " Num epochs = 1\n",
+ " Total optimization steps = 1540\n",
+ " Total train batch size = 16\n"
+ ]
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "POQgkfrWEg1u"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "38bfdddef0444c0baf9d29248689f846",
+ "version_major": 2,
+ "version_minor": 0
},
- "outputs": [],
- "source": [
- "import argilla as rg\n",
- "from datasets import load_dataset\n",
- "from setfit import get_templated_dataset\n",
- "from setfit import SetFitModel, SetFitTrainer"
+ "text/plain": [
+ "Epoch: 0%| | 0/1 [00:00, ?it/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "3. If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "d11aa6a0c8c54481b6cc2c80d1fa0ba1",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Iteration: 0%| | 0/1540 [00:00, ?it/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "topic_model = train_model(\n",
+ " question_name=\"topics\",\n",
+ " template=\"The customer request is about {}\",\n",
+ " multi_label=True,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 276,
+ "referenced_widgets": [
+ "c21e90a6dda643d8bd82abf4e346d45c",
+ "170a2ee20ab64a9b86db65549a5d4063",
+ "fd7c2acc4b1945feabe6715dd270cb72",
+ "2f271b0778974646aaff691227336e91",
+ "ef245777ac3d435e8715fc55b1d4824c",
+ "0d7acd8e1a394336aa146e2a442f672c",
+ "3e6c2b50b3084d23b575585c288f087e",
+ "ff7f98b368c448ea81e4c79fded0be5a",
+ "1ff157a9c8974b07ae97cb115c8d0188",
+ "16d42bc00dfe4467a1da86b1d2391d0d",
+ "0447a98b5dfe42c899273b9c37bdadad",
+ "411de4b297fe4a09acb70951c9f36b82",
+ "c2eac9934f5b407c8e424ee2da9eea58",
+ "36b99521f8274a639abb90eb0040d6c0",
+ "3fd94ef662db4fff9dde61455b41faf1",
+ "d6283b2cf69d45f694633ae1544d47a8",
+ "7ca015b6798947d58d275de6181fe053",
+ "750011ef09534e55bab5180974bcf5d4",
+ "70a57ad580f847d3bd3123cfe1539305",
+ "0c010df989eb497c810a6f960c6ea41b",
+ "186f82d150994ac7914d0646fb5ff425",
+ "379907416f504f05906454e482da2eaf",
+ "783115bacdbf4c0bb09c0b1fc7976d28",
+ "242f97eb0f0d4ab1830c62686127b717",
+ "bfecbc09a4f84f3db51903d5048ff825",
+ "db7cf4427ad746cd86df88f7a1016bc9",
+ "668593b82ae54d3cbaf1a19c0307c545",
+ "5057f8b8144d41ff9d8b82b8602570fc",
+ "369bc409052a48f7ac2182715406abef",
+ "5cc0f7cc30ae4aa4b13966a773e4c824",
+ "28c40914eac34bcba0c9eb4dac6b0032",
+ "3e622eeea5df47d6a21e015f3e742fa8",
+ "621bb7d632814cb0839755ca56098d7a"
+ ]
},
+ "id": "kkTufA4NbEh_",
+ "outputId": "41c579c8-5394-4c24-fd3c-d6ab77c2a0a7"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
- "# Replace api_key if you configured a custom API key\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\", \n",
- " api_key=\"admin.apikey\"\n",
- ")"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "config.json not found in HuggingFace Hub.\n",
+ "WARNING:huggingface_hub.hub_mixin:config.json not found in HuggingFace Hub.\n",
+ "model_head.pkl not found on HuggingFace Hub, initialising classification head with random weights. You should TRAIN this model on a downstream task to use it for predictions and inference.\n"
+ ]
},
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "c21e90a6dda643d8bd82abf4e346d45c",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Generating Training Pairs: 0%| | 0/20 [00:00, ?it/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# # Set the HF_TOKEN environment variable\n",
- "# import os\n",
- "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
- "\n",
- "# # Replace api_url with the url to your HF Spaces URL\n",
- "# # Replace api_key if you configured a custom API key\n",
- "# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
- "# api_key=\"admin.apikey\",\n",
- "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
- "# )"
- ]
+ "name": "stderr",
+ "output_type": "stream",
+ "text": [
+ "***** Running training *****\n",
+ " Num examples = 960\n",
+ " Num epochs = 1\n",
+ " Total optimization steps = 60\n",
+ " Total train batch size = 16\n"
+ ]
},
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Enable Telemetry\n",
- "\n",
- "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "411de4b297fe4a09acb70951c9f36b82",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Epoch: 0%| | 0/1 [00:00, ?it/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
},
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "try:\n",
- " from argilla.utils.telemetry import tutorial_running\n",
- " tutorial_running()\n",
- "except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
+ "data": {
+ "application/vnd.jupyter.widget-view+json": {
+ "model_id": "783115bacdbf4c0bb09c0b1fc7976d28",
+ "version_major": 2,
+ "version_minor": 0
+ },
+ "text/plain": [
+ "Iteration: 0%| | 0/60 [00:00, ?it/s]"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "sentiment_model = train_model(\n",
+ " question_name=\"sentiment\", template=\"This message is {}\", multi_label=False\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Make predictions"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Once the training step is over, we can make predictions over our data."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "def get_predictions(texts, model, question_name):\n",
+ " probas = model.predict_proba(texts, as_numpy=True)\n",
+ " labels = dataset.question_by_name(question_name).labels\n",
+ " for pred in probas:\n",
+ " yield [{\"label\": label, \"score\": score} for label, score in zip(labels, pred)]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "Hz5LeVDMYyx6"
+ },
+ "outputs": [],
+ "source": [
+ "data = data.map(\n",
+ " lambda batch: {\n",
+ " \"topics\": list(get_predictions(batch[\"text\"], topic_model, \"topics\")),\n",
+ " \"sentiment\": list(get_predictions(batch[\"text\"], sentiment_model, \"sentiment\")),\n",
+ " },\n",
+ " batched=True,\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 206
},
+ "id": "bgGkKO-7ZGCR",
+ "outputId": "17bb27eb-b78a-4a2c-d838-60fcaa176502"
+ },
+ "outputs": [
{
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## Configure the dataset"
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
- "\n",
- "Note \n",
- "\n",
- "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
- "\n",
- "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
- "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
- "\n",
- "
"
- ]
+ "cells": [
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "l1RUBJobMBYN"
+ },
+ "source": [
+ "# 🏆 Train a reward model for RLHF\n",
+ "\n",
+ "Collecting comparison data to train a reward model is a crucial part of RLHF and LLM evaluation. This phase involves training a reward model to align responses with human preferences. Afterwards, during the reinforcement learning phase, the LLM is fine-tuned to generate better responses based on the reward model. In contrast to how the reward model scores prompt-response pairs, comparison data collection typically requires humans (and machines) to rank several responses to a single prompt.\n",
+ "\n",
+ "In this example, we will describe how you can **build a dataset for collecting human preferences and train a reward model using the amazing trl library**. See below the workflow we will be following.\n",
+ "\n",
+ "Let's get started!\n",
+ "\n",
+ "\n",
+ "\n",
+ "
\n",
+ "\n",
+ "Note \n",
+ "\n",
+ "This tutorial is a Jupyter Notebook. There are two options to run it:\n",
+ "\n",
+ "- Use the Open in Colab button at the top of this page. This option allows you to run the notebook directly on Google Colab. Don't forget to change the runtime type to GPU for faster model training and inference.\n",
+ "- Download the .ipynb file by clicking on the View source link at the top of the page. This option allows you to download the notebook and run it on your local machine or on a Jupyter notebook tool of your choice.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "hZb87bUJMBYQ"
+ },
+ "source": [
+ "## Setup\n",
+ "\n",
+ "For this tutorial, you will need to have an Argilla server running. If you don't have one already, check out our [Quickstart](../../../getting_started/quickstart.md) or [Installation](../../../getting_started/quickstart_installation.ipynb) pages. Once you do, complete the following steps:\n",
+ "\n",
+ "1. Install the Argilla client and the required third-party libraries using `pip`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "dSw4HgoJMBYT"
+ },
+ "outputs": [],
+ "source": [
+ "%pip install -U argilla pandas trl plotly -qqq"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "JoVCJkSlMBYT"
+ },
+ "source": [
+ "2. Let's make the necessary imports:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 1,
+ "metadata": {
+ "id": "IbSU2uDhMBYU"
+ },
+ "outputs": [],
+ "source": [
+ "import random\n",
+ "\n",
+ "import torch\n",
+ "from datasets import Dataset, load_dataset\n",
+ "from transformers import (\n",
+ " AutoModelForSequenceClassification,\n",
+ " AutoTokenizer,\n",
+ " TrainingArguments,\n",
+ ")\n",
+ "from trl import RewardTrainer\n",
+ "\n",
+ "import argilla as rg"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "A5gHXENrMBYW"
+ },
+ "source": [
+ "3. If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "DZRJUFUhMBYW"
+ },
+ "outputs": [],
+ "source": [
+ "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
+ "# you can find the Spaces URL under the Embed this space button\n",
+ "# Replace api_key if you configured a custom API key\n",
+ "rg.init(api_url=\"http://localhost:6900\", api_key=\"admin.apikey\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# # Set the HF_TOKEN environment variable\n",
+ "# import os\n",
+ "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
+ "\n",
+ "# # Replace api_url with the url to your HF Spaces URL\n",
+ "# # Replace api_key if you configured a custom API key\n",
+ "# rg.init(\n",
+ "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\",\n",
+ "# api_key=\"admin.apikey\",\n",
+ "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
+ "# )"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "### Enable Telemetry\n",
+ "\n",
+ "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "try:\n",
+ " from argilla.utils.telemetry import tutorial_running\n",
+ "\n",
+ " tutorial_running()\n",
+ "except ImportError:\n",
+ " print(\n",
+ " \"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\"\n",
+ " )"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "wmT7RQqlMBYW"
+ },
+ "source": [
+ "## Configure the dataset\n",
+ "As a first step, let's load the dataset and quickly explore the data. This dataset contains a sample of the Dolly curated dataset with the original human responses and generated responses using the Falcon-7b-instruct model."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "hZb87bUJMBYQ"
- },
- "source": [
- "## Setup\n",
- "\n",
- "For this tutorial, you will need to have an Argilla server running. If you don't have one already, check out our [Quickstart](../../../getting_started/quickstart.md) or [Installation](../../../getting_started/quickstart_installation.ipynb) pages. Once you do, complete the following steps:\n",
- "\n",
- "1. Install the Argilla client and the required third-party libraries using `pip`:"
- ]
+ "id": "tYpQbwORMBYX",
+ "outputId": "07f3808f-5965-4cb9-9375-0d3e92c2436d"
+ },
+ "outputs": [],
+ "source": [
+ "hf_dataset = load_dataset(\n",
+ " \"argilla/dolly-curated-comparison-falcon-7b-instruct\", split=\"train\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 780
},
+ "id": "61DMAbebMBYY",
+ "outputId": "97009132-c55e-4b64-9dc3-bb922b74e414"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "dSw4HgoJMBYT"
- },
- "outputs": [],
- "source": [
- "%pip install -U argilla pandas trl plotly -qqq"
+ "data": {
+ "text/html": [
+ "
\n",
+ "\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
prompt
\n",
+ "
response-1
\n",
+ "
response-2
\n",
+ "
category
\n",
+ "
original_response
\n",
+ "
external_id
\n",
+ "
\n",
+ " \n",
+ " \n",
+ "
\n",
+ "
0
\n",
+ "
What is Depreciation
\n",
+ "
What is Depreciation – 10 Important Facts to K...
\n",
+ "
What is Depreciation on a Car?\\nDepreciation i...
\n",
+ "
open_qa
\n",
+ "
Depreciation is the drop in value of an asset ...
\n",
+ "
518
\n",
+ "
\n",
+ "
\n",
+ "
1
\n",
+ "
What do you know about the city of Aberdeen in...
\n",
+ "
Aberdeen, the city in which I've lived for the...
\n",
+ "
As an AI language model, I don't have personal...
\n",
+ "
open_qa
\n",
+ "
Aberdeen is a city located in the North East o...
\n",
+ "
351
\n",
+ "
\n",
+ "
\n",
+ "
2
\n",
+ "
Describe thunderstorm season in the United Sta...
\n",
+ "
Describe thunderstorm season in the United Sta...
\n",
+ "
Describe thunderstorm season in the United Sta...
\n",
+ "
information_extraction
\n",
+ "
Thunderstorm season in the United States and C...
\n",
+ "
10567
\n",
+ "
\n",
+ "
\n",
+ "
3
\n",
+ "
When did Peloton IPO?\\nOn September 26, 2019, ...
\n",
+ "
When did Peloton IPO?\\nPeloton launched its in...
\n",
+ "
When did Peloton IPO?\\nPeloton IPO'd on May 26...
\n",
+ "
closed_qa
\n",
+ "
Peloton became a public company via an initial...
\n",
+ "
12412
\n",
+ "
\n",
+ "
\n",
+ "
4
\n",
+ "
What is the best way to answer an interview qu...
\n",
+ "
The best way to answer an interview question m...
\n",
+ "
Some of the best ways to answer an interview q...
\n",
+ "
creative_writing
\n",
+ "
The first recommended step is to ask clarifyin...
\n",
+ "
2521
\n",
+ "
\n",
+ "
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
...
\n",
+ "
\n",
+ "
\n",
+ "
7396
\n",
+ "
How do i accept the change
\n",
+ "
How do i accept the change in my life\\nAccepti...
\n",
+ "
I's a great opportunity to improve. The only t...
\n",
+ "
brainstorming
\n",
+ "
Embrace the change and see the difference
\n",
+ "
15010
\n",
+ "
\n",
+ "
\n",
+ "
7397
\n",
+ "
Extract the teams that the footballer Sócrates...
\n",
+ "
Extract the teams that the footballer Sócrates...
\n",
+ "
Extract the teams that the footballer Sócrates...
\n",
+ "
information_extraction
\n",
+ "
Brazil, Botafogo-SP, Corinthians, Fiorentina
\n",
+ "
9970
\n",
+ "
\n",
+ "
\n",
+ "
7398
\n",
+ "
Without quoting directly from the text give me...
\n",
+ "
Without quoting directly from the text give me...
\n",
+ "
Without quoting directly from the text give me...
\n",
+ "
summarization
\n",
+ "
Brendon Small is a stand-up comedian, Creator...
\n",
+ "
14205
\n",
+ "
\n",
+ "
\n",
+ "
7399
\n",
+ "
Is Killing is Sin ? Is it ture
\n",
+ "
Is Killing is Sin ? Is it ture?\\nKilling can b...
\n",
+ "
Is Killing is Sin ? Is it ture?\\nKilling is no...
\n",
+ "
brainstorming
\n",
+ "
Killing a human being should not be sin becaus...
\n",
+ "
11253
\n",
+ "
\n",
+ "
\n",
+ "
7400
\n",
+ "
Who was Otto von Bismarck?\\nOtto, Prince of Bi...
\n",
+ "
Who was Otto von Bismarck?\\nOtto von Bismarck ...
\n",
+ "
Who was Otto von Bismarck?\\nOtto von Bismarck ...
\n",
+ "
information_extraction
\n",
+ "
Otto von Bismarck was a Prussian and German so...
\n",
+ "
12872
\n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
7401 rows × 6 columns
\n",
+ "
"
+ ],
+ "text/plain": [
+ " prompt \\\n",
+ "0 What is Depreciation \n",
+ "1 What do you know about the city of Aberdeen in... \n",
+ "2 Describe thunderstorm season in the United Sta... \n",
+ "3 When did Peloton IPO?\\nOn September 26, 2019, ... \n",
+ "4 What is the best way to answer an interview qu... \n",
+ "... ... \n",
+ "7396 How do i accept the change \n",
+ "7397 Extract the teams that the footballer Sócrates... \n",
+ "7398 Without quoting directly from the text give me... \n",
+ "7399 Is Killing is Sin ? Is it ture \n",
+ "7400 Who was Otto von Bismarck?\\nOtto, Prince of Bi... \n",
+ "\n",
+ " response-1 \\\n",
+ "0 What is Depreciation – 10 Important Facts to K... \n",
+ "1 Aberdeen, the city in which I've lived for the... \n",
+ "2 Describe thunderstorm season in the United Sta... \n",
+ "3 When did Peloton IPO?\\nPeloton launched its in... \n",
+ "4 The best way to answer an interview question m... \n",
+ "... ... \n",
+ "7396 How do i accept the change in my life\\nAccepti... \n",
+ "7397 Extract the teams that the footballer Sócrates... \n",
+ "7398 Without quoting directly from the text give me... \n",
+ "7399 Is Killing is Sin ? Is it ture?\\nKilling can b... \n",
+ "7400 Who was Otto von Bismarck?\\nOtto von Bismarck ... \n",
+ "\n",
+ " response-2 \\\n",
+ "0 What is Depreciation on a Car?\\nDepreciation i... \n",
+ "1 As an AI language model, I don't have personal... \n",
+ "2 Describe thunderstorm season in the United Sta... \n",
+ "3 When did Peloton IPO?\\nPeloton IPO'd on May 26... \n",
+ "4 Some of the best ways to answer an interview q... \n",
+ "... ... \n",
+ "7396 I's a great opportunity to improve. The only t... \n",
+ "7397 Extract the teams that the footballer Sócrates... \n",
+ "7398 Without quoting directly from the text give me... \n",
+ "7399 Is Killing is Sin ? Is it ture?\\nKilling is no... \n",
+ "7400 Who was Otto von Bismarck?\\nOtto von Bismarck ... \n",
+ "\n",
+ " category \\\n",
+ "0 open_qa \n",
+ "1 open_qa \n",
+ "2 information_extraction \n",
+ "3 closed_qa \n",
+ "4 creative_writing \n",
+ "... ... \n",
+ "7396 brainstorming \n",
+ "7397 information_extraction \n",
+ "7398 summarization \n",
+ "7399 brainstorming \n",
+ "7400 information_extraction \n",
+ "\n",
+ " original_response external_id \n",
+ "0 Depreciation is the drop in value of an asset ... 518 \n",
+ "1 Aberdeen is a city located in the North East o... 351 \n",
+ "2 Thunderstorm season in the United States and C... 10567 \n",
+ "3 Peloton became a public company via an initial... 12412 \n",
+ "4 The first recommended step is to ask clarifyin... 2521 \n",
+ "... ... ... \n",
+ "7396 Embrace the change and see the difference 15010 \n",
+ "7397 Brazil, Botafogo-SP, Corinthians, Fiorentina 9970 \n",
+ "7398 Brendon Small is a stand-up comedian, Creator... 14205 \n",
+ "7399 Killing a human being should not be sin becaus... 11253 \n",
+ "7400 Otto von Bismarck was a Prussian and German so... 12872 \n",
+ "\n",
+ "[7401 rows x 6 columns]"
]
+ },
+ "execution_count": 8,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "df = hf_dataset.to_pandas()\n",
+ "df"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "-OMxM53bMBYZ"
+ },
+ "source": [
+ "For reward modeling, we would like to ask labelers to rank two responses for a prompt from best to worst. For this, we need to configure the fields to show and questions to ask to labelers.\n",
+ "\n",
+ "The dataset will show the users three fields `instruction`, which corresponds to the prompt, `response-1` and `response-2`, which correspond to each of the responses to the instruction.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {
+ "id": "VZ5prVQNMBYa"
+ },
+ "outputs": [],
+ "source": [
+ "# list of fields that we will use later for our dataset settings\n",
+ "fields = [\n",
+ " rg.TextField(name=\"instruction\", title=\"User instruction\"),\n",
+ " rg.TextField(name=\"response-1\"),\n",
+ " rg.TextField(name=\"response-2\"),\n",
+ "]"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "VKU888AnMBYa"
+ },
+ "source": [
+ "In this use case, we will ask the users to select the best response. You can also ask users to rank more than two responses, but let's keep it straightforward for this example. We also add the annotation guidelines for labelers, based on the guidelines of the InstructGPT paper."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {
+ "id": "wJvW18MEMBYb"
+ },
+ "outputs": [],
+ "source": [
+ "question = rg.RatingQuestion(\n",
+ " name=\"choose-best\",\n",
+ " title=\"Choose the best response:\",\n",
+ " description=\"Choose the most helpful, harmless, and truthful response. Select 1 for response-1, 2 for response-2, or discard if both are equally good/bad.\",\n",
+ " values=[1, 2],\n",
+ " required=True,\n",
+ ")\n",
+ "\n",
+ "guidelines = \"\"\"These guidelines are based on the paper [Training Language Models to Follow Instructions with Human Feedback](https://cdn.openai.com/papers/Training_language_models_to_follow_instructions_with_human_feedback.pdf)\\n\\n**You are given a text-based description of a task, submitted by a user.**\\n\\nThis task description may be in the form of an explicit instruction (e.g. \\\"Write a story about a wise frog.\\\"). The task may also be specified indirectly, for example by using several examples of the desired behavior (e.g. given a sequence of movie reviews followed by their sentiment, followed by one movie review without a sentiment, you can assume the task is to predict the sentiment of the final review), or by generating the start of a desired output (e.g. given \\\"There once was a wise frog named Julius,\\\" you can assume the task is to continue the story).\\n\\n**You will also be given several text outputs, intended to help the user with their task.** Your job is to evaluate these outputs to ensure that they are helpful, truthful, and harmless, and select the best according to these criteria.\\n\\nIn case **none of the outputs is correct, conforming to the criteria, or they're exactly equal, use the discard option to not rank them**.\\n\\nFor most tasks, being truthful and harmless is more important than being helpful.\\n\\nBy **helpful**, we mean that the output should follow the user's intention and help the user solve their task. Some examples of helpful behavior:\\n\\n- Writing in clear language.\\n- Answering the question they meant to ask, even if they mis-asked it.\\n- Being sensitive to internationality (e.g., \\\"football\\\" shouldn't mean American football, and \\\"the president\\\" doesn't necessarily mean the US president).\\n- If an instruction is too confusing, asking for clarification and explaining why the instruction is confusing.\\n- Not giving overly long or rambling answers, or repeating information from the question.\\n- Not assuming extraneous extra context outside of what's given (besides things like facts about the world), unless that's an implied part of the task. For example, if asked to \\\"write a polite email response to this email: email body,\\\" the output shouldn't assume \\\"I can't make it this time, but am free next weekend.\\\" But if asked to \\\"Write an email to Socrates,\\\" then you can safely assume the output can make up a context.\\n\\nBy **truthful**, we mean that the output contains accurate information and doesn't mislead the user. Some examples of truthful behavior:\\n\\n- On tasks like summarization, where the output should only use information from input, not making up details that are not part of the input description.\\n- Not producing clearly false information about the world (e.g., making up facts or promoting conspiracies). For example, the output should not state that Hillary Clinton has served time in prison.\\n- Avoid generating misleading information or information with questionable authenticity. For example, if the input asks \\\"Why did Hillary Clinton go to jail?\\\", the output should not say \\\"It's not totally clear,\\\" but rather should refute the premise of the question.\\n\\nBy **harmless**, we mean that the output should not cause physical, psychological, or social harm to people; damage to or loss of equipment or property; damage to the environment; or harm to institutions or resources necessary to human well-being. Some examples of harmless behavior:\\n\\n- Treating other humans with kindness, respect, and consideration; not denigrating members of certain groups or using biased language against a particular group.\\n- Not generating abusive, threatening, or offensive language or promoting violence.\\n- Not writing sexual or violent content if it's not asked for.\\n- Not giving bad real-world advice or promoting illegal activity.\\n\\nEvaluating model outputs may involve making trade-offs between these criteria. These trade-offs will depend on the task.\"\"\""
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "RfZuuRpr07n4"
+ },
+ "source": [
+ "## Create records\n",
+ "\n",
+ "The next step is to build the records for collecting comparison data. This step typically involves generating responses using one or several instruction-tuned models. \n",
+ "\n",
+ ".. tip:: When showing responses from two different models to labelers it's recommended to randomly assign different model responses to `response-1` and `response-2` for each record. Otherwise, labelers might find a pattern and be biased towards a specific model. This is especially relevant for model comparison and evaluation but also applies to comparison data for reward modeling. \n",
+ "\n",
+ "In this example, we've already generated a dataset using the instructions from the Dolly curated dataset with the Falcon-7B-instruct model. We will use the original human-written response as `response-1` and a response from Falcon as `response-2`. \n",
+ "\n",
+ "You can build the records and publish them for labelers as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "JoVCJkSlMBYT"
- },
- "source": [
- "2. Let's make the necessary imports:"
- ]
+ "id": "qZUeAT-1OFfz",
+ "outputId": "2b605fd6-3708-4f8c-d18a-4dc1f75f981e"
+ },
+ "outputs": [],
+ "source": [
+ "# build records from hf dataset\n",
+ "records = [\n",
+ " rg.FeedbackRecord(\n",
+ " fields={\n",
+ " \"instruction\": r[\"prompt\"],\n",
+ " \"response-1\": r[\"original_response\"],\n",
+ " \"response-2\": r[\"response-2\"],\n",
+ " }\n",
+ " )\n",
+ " for r in hf_dataset\n",
+ "]\n",
+ "\n",
+ "# create dataset\n",
+ "dataset = rg.FeedbackDataset(fields=fields, questions=[question], guidelines=guidelines)\n",
+ "\n",
+ "# add records and publish\n",
+ "dataset.add_records(records)\n",
+ "dataset.push_to_argilla(\"comparison-data-falcon\", workspace=\"admin\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Now the dataset is ready for labeling. This is the Feedback UI we have just configured:\n",
+ "\n",
+ "\n",
+ "\n",
+ "\n"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "4vUhGWWnufDq"
+ },
+ "source": [
+ "Additionally, you can push the dataset to the Hub for reproducibility and reuse. This dataset is available in the Hub, feel free to [read the dataset card](https://huggingface.co/datasets/argilla/comparison-data-falcon-with-feedback) to understand its structure, annotation guidelines, and how to import it."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "0kqc8fADoX4Q"
+ },
+ "outputs": [],
+ "source": [
+ "# dataset.push_to_huggingface(\"comparison-data-falcon\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "aW40aC5xvhrR"
+ },
+ "source": [
+ "## Collect feedback and prepare the dataset\n",
+ "\n",
+ "Once the data has been labeled using the Argilla UI, we can retrieve it with the Python SDK and prepare it for training the reward model with TRL.\n",
+ "\n",
+ "If you are running this tutorial but haven´t labeled any data points, execute the following cell to retrieve the labeled dataset from the Hugging Face Hub. This dataset already contains ranked responses and can be used for the next steps. The dataset is available in the Hub, feel free to [read the dataset card](https://huggingface.co/datasets/argilla/comparison-data-falcon-with-feedback) to understand its structure, annotation guidelines, responses, and how to import it. \n",
+ "\n",
+ "If you have labeled some examples jump and execute the second cell. "
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 238,
+ "referenced_widgets": [
+ "b4a7efcf448247c586546cee7ae4e853",
+ "ad4dd7f68c00437e878e2965e08457c8",
+ "e7b96b86f6df4826b04171eaec7fa2a6",
+ "56021dd10ea447029b2d64a3954f50a8",
+ "ff6274caa2cb4346b9862dcbf1846184",
+ "a681213d4f1f4c0db8cf2be709734067",
+ "626d87ba234a4af9aed53f633cefbd95",
+ "1c28d5ed9c7a41ad87e7fd83380e05a5",
+ "dab35d7560824b828e1976691e481301",
+ "23cccf0f1f2240ecaabb5d6efa7a8bbb",
+ "f5d4cf0d204f478094976a30309de13e",
+ "1eca0bb4151543098cfbb98b64191f92",
+ "299f4ae7c6524b9d8a06f22063d5a92f",
+ "4ed8d36201904c6cb43d0fcdf7b18f34",
+ "b9ed9f3bcbfd4403b4c1ea9eb3001543",
+ "a45c853e3a7c4b65ad3bdae9a67607bc",
+ "fc422a04dbcf4b979fa7569c171ac971",
+ "a291ef8013ee445b8f3e36be30e1da69",
+ "7ba0eca5e7984391b4d32608c85bd236",
+ "0e746ea7187c4b7382de0e5094d6234a",
+ "324c48fb64fb4021a4acec428fcf53ac",
+ "841a58e70b3f4621a4419d2872b3d348",
+ "3cc6da03d28447d0a7259c460b017296",
+ "ffc2b422e5284a3180e703cab2ebef1e",
+ "5151b3406f774c57b66c0651e10edb42",
+ "b04603b9ced743d991a61e42fc0d8ce6",
+ "99f75e3565334a0cad80b0ddf434156c",
+ "e295f279b61f48d683784131a8d533cc",
+ "254c7bb9621c457182976287a0a5ac66",
+ "9e60f5105a2f430d9c5f382c54b0610c",
+ "2b34ee7a285a4dbb8c1001d8162b8e44",
+ "4d451a340eb341ee88002e9ecce1d2a9",
+ "8c1c4cac8f0444e4a69bd316efde9080",
+ "b3dff83de3894147838358514c8ef63c",
+ "ac4deaed71894ca9b8bdba7e14a34212",
+ "8ed4b17c0dc4464cb5cc8bcfbc6bc803",
+ "9a45be3cbddc4d0cbcc3591fe2c2883e",
+ "51d0e514f6244692b2b1e6dbfcc0a460",
+ "0432b133dc614d1fa8e2d06edc413302",
+ "ffe071addb80421dba36b5797a594dbe",
+ "17ed61c237304d2bb77b80bd4def6466",
+ "809240690fb44738b6173cc53504bcf1",
+ "5d3a83515f8944d491fe3305ee01a314",
+ "52d358e242dd4876a66adbdb89fb03fc",
+ "dd782a3398d1437f9d3626bdf4b87be4",
+ "f632551058f744b0b9ad9696397c8d53",
+ "914b7ab6f21d486fa42be37545ea73d7",
+ "94903b2e56ef43b9b39ddd576a5a06ee",
+ "902a2c7722484d4283e5ba0757641a0e",
+ "23ef7fb2393f45d885d11ada8e5a3534",
+ "3aaa39cccc26416f9c7aeab4d50da8c5",
+ "04c0309dc09e46419995addb7238f5ee",
+ "a9dcbe78ae6641d38775818748dd71c4",
+ "5c23c54cc6bd4475ba755490afb520db",
+ "bda482b92a8d4229983e72581154eba9"
+ ]
},
- {
- "cell_type": "code",
- "execution_count": 1,
- "metadata": {
- "id": "IbSU2uDhMBYU"
- },
- "outputs": [],
- "source": [
- "import random\n",
- "\n",
- "import torch\n",
- "from datasets import Dataset, load_dataset\n",
- "from transformers import (\n",
- " AutoModelForSequenceClassification,\n",
- " AutoTokenizer,\n",
- " TrainingArguments,\n",
- ")\n",
- "from trl import RewardTrainer\n",
- "\n",
- "import argilla as rg"
- ]
+ "id": "p-0_iwa8IA2y",
+ "outputId": "abbc3e3a-6053-4a0e-c1a1-f3f5f32d686c"
+ },
+ "outputs": [],
+ "source": [
+ "# if you haven't ranked any responses with the UI run this cell\n",
+ "# otherwise ran the next one\n",
+ "feedback_dataset = rg.FeedbackDataset.from_huggingface(\n",
+ " \"argilla/comparison-data-falcon-with-feedback\"\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/"
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "A5gHXENrMBYW"
- },
- "source": [
- "3. If you are running Argilla using the Docker quickstart image or Hugging Face Spaces, you need to init the Argilla client with the `URL` and `API_KEY`:"
- ]
+ "id": "TZGfBzgmvg1d",
+ "outputId": "7efee8d2-79b2-4555-a489-f8a55fcbec31"
+ },
+ "outputs": [],
+ "source": [
+ "# run this cell if you have ranked the responses in the UI\n",
+ "feedback_dataset = rg.FeedbackDataset.from_argilla(\n",
+ " \"comparison-data-falcon\", workspace=\"admin\"\n",
+ ")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The next step is to prepare the dataset in the standard format for training a reward model. In particular, we want to select the chosen and rejected responses from the user feedback. We do this by creating a `TrainingTask` instance for reward modeling using a function that returns chosen-rejected tuples."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 3,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 537
},
+ "id": "cll7j10swNT2",
+ "outputId": "0e4974e6-74b1-41be-c54c-92bca9732ddc"
+ },
+ "outputs": [],
+ "source": [
+ "from typing import Any, Dict\n",
+ "from argilla.feedback import TrainingTask\n",
+ "from collections import Counter\n",
+ "\n",
+ "\n",
+ "def formatting_func(sample: Dict[str, Any]):\n",
+ " # sample[\"choose-best\"] => [{'user_id': None, 'value': 1, 'status': 'submitted'}, ...]\n",
+ " values = [\n",
+ " annotation[\"value\"]\n",
+ " for annotation in sample[\"choose-best\"]\n",
+ " if annotation[\"status\"] == \"submitted\"\n",
+ " ]\n",
+ "\n",
+ " # We will only focus on the annotated records in the dataset\n",
+ " if Counter(values).most_common(1) != []:\n",
+ " # values => [1]\n",
+ " winning_response = Counter(values).most_common(1)[0][0]\n",
+ " print(Counter(values).most_common(1))\n",
+ " if winning_response == 1:\n",
+ " chosen = sample[\"response-1\"]\n",
+ " rejected = sample[\"response-2\"]\n",
+ " else:\n",
+ " chosen = sample[\"response-2\"]\n",
+ " rejected = sample[\"response-1\"]\n",
+ " return chosen, rejected\n",
+ "\n",
+ "\n",
+ "task = TrainingTask.for_reward_modeling(formatting_func=formatting_func)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "If we want, we can observe the resulting dataset by preparing it for training with TRL:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 4,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "DZRJUFUhMBYW"
- },
- "outputs": [],
- "source": [
- "# Replace api_url with the url to your HF Spaces URL if using Spaces\n",
- "# you can find the Spaces URL under the Embed this space button\n",
- "# Replace api_key if you configured a custom API key\n",
- "rg.init(\n",
- " api_url=\"http://localhost:6900\",\n",
- " api_key=\"admin.apikey\"\n",
- ")"
+ "data": {
+ "text/plain": [
+ "Dataset({\n",
+ " features: ['chosen', 'rejected'],\n",
+ " num_rows: 7401\n",
+ "})"
]
- },
+ },
+ "execution_count": 4,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "dataset = feedback_dataset.prepare_for_training(framework=\"trl\", task=task)\n",
+ "dataset"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 5,
+ "metadata": {},
+ "outputs": [
{
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If you're running a private Hugging Face Space, you will also need to set the [HF_TOKEN](https://huggingface.co/settings/tokens) as follows:"
+ "data": {
+ "text/plain": [
+ "{'chosen': \"Depreciation is the drop in value of an asset due to wear and tear, age and obsolescence (going out of date) as recorded in an organization's financial records.\",\n",
+ " 'rejected': 'What is Depreciation – 10 Important Facts to Know?\\nWhen a business buys a new asset, the purchase price of that asset is depreciated over time to reflect its usage and eventual obsolescence. Depreciation expense can be a tax deductible expense and is usually a non-cash expense reported on a company’s income statement and balance sheet. The amount of depreciation expense a company reports each year is the difference between the original purchase price of the asset and what the current value of that asset might be. Here are 10 important facts to know about depreciation:\\n1. Depreciation is a non-cash expense. It is an expense that is reported in a business’s income statement and balance sheet and not a cash flow expense.\\n2. Depreciation is an accounting standard and it is required to be disclosed in a business’s financial statements.\\n3. The amount of depreciation is usually a tax expense and not a cash expense reported on a company’s income statement'}"
]
+ },
+ "execution_count": 5,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "dataset[0]"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "UgoIyJanMBYk"
+ },
+ "source": [
+ "This dataset is ready to be used as comparison data to train a reward model.\n",
+ "\n",
+ "
\n",
+ "\n",
+ "Note\n",
+ "\n",
+ "The paper Direct Preference Optimization: Your Language Model is Secretly a Reward Model proposes DPO, a promising method for using comparison data directly to model human preference, eliminating the need for a reward model and the RL step. Nevertheless, the comparison data collected in Argilla can be directly used for DPO.\n",
+ "\n",
+ "
"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "F8r9_XJi1K9P"
+ },
+ "source": [
+ "## Train the reward model with `trl`\n",
+ "In this step, we will use the `RewardTrainer` from the trl library. To understand this step, we recommend you to [check the trl docs](https://huggingface.co/docs/trl/reward_trainer). \n",
+ "\n",
+ "
\n",
+ "To run this step, you need to rank some examples using the Argilla UI, or run the step above with the load from Hugging Face call: `feedback_dataset = FeedbackDataset.from_huggingface`\n",
+ "
\n",
+ "\n",
+ "To train a Reward Model, you need to choose a base model to fine-tune. In the literature, the base model is typically the supervised fine-tuned model resulting from the instruction-tuning step. In this example, that would mean using the [Falcon-7B-instruct model](https://huggingface.co/tiiuae/falcon-7b-instruct). However, as Reward Models are essentially classifiers you can use a more lightweight backbone model, for this example we will use `distilroberta-base` but feel free to experiment with other models.\n",
+ "\n",
+ "The code below fine-tunes a `SequenceClassification` model with our preference dataset. The most interesting part is the `formatting_func` function. This function combines instructions with chosen and rejected responses, creating two new strings. These strings are tokenized, becoming input for a reward model that learns to distinguish between good and bad responses based on these examples. The model will be optimized to assign higher values to preferred responses and lower values to rejected responses.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 7,
+ "metadata": {
+ "colab": {
+ "base_uri": "https://localhost:8080/",
+ "height": 516,
+ "referenced_widgets": [
+ "b44d62c0bef242fabab85ff6129b42dc",
+ "9a872753cf9a4fde9caa69660e85c896",
+ "2a9d2460cf024776bb0010ff6f1a5fa7",
+ "8a6d78da941d429b981d82a1b0d249ab",
+ "8b2770c23c424f8698d7f3ca60bf5d9a",
+ "9fc9251942db44548dcc9b55312f6963",
+ "660b25b7377a4bd0a2baf03a437dc060",
+ "6e00a979fa604747b320018d4d5549ad",
+ "5ed78905729747d8a3dfe68590950d73",
+ "fdd275ff03504203b5dbce2ea732fd44",
+ "7d01e2401fa44199afb31d5dcc443761"
+ ]
},
+ "id": "-MtSUam61J_Z",
+ "outputId": "6680758c-3b9c-444a-e73b-48bbb4c1b4b5"
+ },
+ "outputs": [
{
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# # Set the HF_TOKEN environment variable\n",
- "# import os\n",
- "# os.environ['HF_TOKEN'] = \"your-hf-token\"\n",
- "\n",
- "# # Replace api_url with the url to your HF Spaces URL\n",
- "# # Replace api_key if you configured a custom API key\n",
- "# rg.init(\n",
- "# api_url=\"https://[your-owner-name]-[your_space_name].hf.space\", \n",
- "# api_key=\"admin.apikey\",\n",
- "# extra_headers={\"Authorization\": f\"Bearer {os.environ['HF_TOKEN']}\"},\n",
- "# )"
+ "data": {
+ "text/html": [
+ "
\n"
+ ],
+ "text/plain": [
+ "\u001b[2;36m[08/08/23 16:36:51]\u001b[0m\u001b[2;36m \u001b[0m\u001b[34mINFO \u001b[0m INFO:ArgillaTRLTrainer:\u001b[1m{\u001b[0m\u001b[32m'eval_loss'\u001b[0m: \u001b[1;36m0.1626577377319336\u001b[0m, \u001b[32m'eval_accuracy'\u001b[0m: \u001b]8;id=234053;file://C:\\code\\argilla\\src\\argilla\\client\\feedback\\training\\frameworks\\trl.py\u001b\\\u001b[2mtrl.py\u001b[0m\u001b]8;;\u001b\\\u001b[2m:\u001b[0m\u001b]8;id=146316;file://C:\\code\\argilla\\src\\argilla\\client\\feedback\\training\\frameworks\\trl.py#226\u001b\\\u001b[2m226\u001b[0m\u001b]8;;\u001b\\\n",
+ "\u001b[2;36m \u001b[0m \u001b[1;36m0.937204591492235\u001b[0m, \u001b[32m'eval_runtime'\u001b[0m: \u001b[1;36m6.5907\u001b[0m, \u001b[32m'eval_samples_per_second'\u001b[0m: \u001b[2m \u001b[0m\n",
+ "\u001b[2;36m \u001b[0m \u001b[1;36m224.709\u001b[0m, \u001b[32m'eval_steps_per_second'\u001b[0m: \u001b[1;36m28.221\u001b[0m, \u001b[32m'epoch'\u001b[0m: \u001b[1;36m1.0\u001b[0m\u001b[1m}\u001b[0m \u001b[2m \u001b[0m\n"
]
+ },
+ "metadata": {},
+ "output_type": "display_data"
+ }
+ ],
+ "source": [
+ "from argilla.feedback import ArgillaTrainer\n",
+ "\n",
+ "model_name = \"distilroberta-base\"\n",
+ "trainer = ArgillaTrainer(\n",
+ " dataset=feedback_dataset,\n",
+ " task=task,\n",
+ " framework=\"trl\",\n",
+ " model=model_name,\n",
+ " train_size=0.8,\n",
+ ")\n",
+ "trainer.update_config(\n",
+ " per_device_train_batch_size=16,\n",
+ " evaluation_strategy=\"steps\",\n",
+ " logging_steps=200,\n",
+ ")\n",
+ "trainer.train(\"./reward_model\")"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Using the Reward Model\n",
+ "The resulting model is [fully open-source and available on the Hugging Hub](https://huggingface.co/argilla/roberta-base-reward-model-falcon-dolly).\n",
+ "\n",
+ "\n",
+ "This is how you can use it with your own data:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
+ "\n",
+ "tokenizer = AutoTokenizer.from_pretrained(\n",
+ " \"argilla/roberta-base-reward-model-falcon-dolly\"\n",
+ ")\n",
+ "\n",
+ "model = AutoModelForSequenceClassification.from_pretrained(\n",
+ " \"argilla/roberta-base-reward-model-falcon-dolly\"\n",
+ ")\n",
+ "\n",
+ "\n",
+ "def get_score(model, tokenizer, prompt, response):\n",
+ " # Tokenize the input sequences\n",
+ " inputs = tokenizer.encode_plus(\n",
+ " prompt,\n",
+ " response,\n",
+ " truncation=True,\n",
+ " padding=\"max_length\",\n",
+ " max_length=512,\n",
+ " return_tensors=\"pt\",\n",
+ " )\n",
+ "\n",
+ " # Perform forward pass\n",
+ " with torch.no_grad():\n",
+ " outputs = model(**inputs)\n",
+ "\n",
+ " # Extract the logits\n",
+ " logits = outputs.logits\n",
+ "\n",
+ " return logits.item()\n",
+ "\n",
+ "\n",
+ "# Example usage\n",
+ "prompt = \"What is Depreciation\"\n",
+ "example_less_pref_response = \"What is Depreciation – 10 Important Facts to Know? When a business buys a new asset, the purchase price of that asset is depreciated over time to reflect its usage and eventual obsolescence. Depreciation expense can be a tax deductible expense and is usually a non-cash expense reported on a company’s income statement and balance sheet. The amount of depreciation expense a company reports each year is the difference between the original purchase price of the asset and what the current value of that asset might be. Here are 10 important facts to know about depreciation: 1. Depreciation is a non-cash expense. It is an expense that is reported in a business’s income statement and balance sheet and not a cash flow expense. 2. Depreciation is an accounting standard and it is required to be disclosed in a business’s financial statements. 3. The amount of depreciation is usually a tax expense and not a cash expense reported on a company’s income statement\"\n",
+ "example_preferred_response = \"Depreciation is the drop in value of an asset due to wear and tear, age and obsolescence (going out of date) as recorded in an organization's financial records.\"\n",
+ "\n",
+ "score = get_score(model, tokenizer, prompt, example_less_pref_response)\n",
+ "print(score)\n",
+ "# >> -3.915163993835449\n",
+ "\n",
+ "score = get_score(model, tokenizer, prompt, example_preferred_response)\n",
+ "print(score)\n",
+ "# >> 7.460323333740234"
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "cxy90ZY4MBYk"
+ },
+ "source": [
+ "## Summary\n",
+ "\n",
+ "In this tutorial, we learned how to create a comparison dataset by ranking responses from the Dolly dataset and Falcon. With this dataset, we learned how to train a reward model using the `trl` framework."
+ ]
+ },
+ {
+ "attachments": {},
+ "cell_type": "markdown",
+ "metadata": {
+ "id": "BeKXm3s3IFr9"
+ },
+ "source": [
+ "## Appendix: How to build the dataset with pre-loaded responses"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {
+ "id": "efyVbRYXIL6H"
+ },
+ "outputs": [],
+ "source": [
+ "picker = [\"response-1\", \"response-2\"]\n",
+ "\n",
+ "\n",
+ "def get_chosen_and_not_chosen(l):\n",
+ " # Generate a random index between 0 and length of the list - 1\n",
+ " chosen_id = random.randint(0, len(l) - 1)\n",
+ " not_chosen_id = 1 - chosen_id # This will be 0 if chosen_id is 1 and vice versa\n",
+ "\n",
+ " return l[chosen_id], l[not_chosen_id], chosen_id\n",
+ "\n",
+ "\n",
+ "records = []\n",
+ "\n",
+ "for r in hf_dataset:\n",
+ " chosen, not_chosen, chosen_id = get_chosen_and_not_chosen(picker)\n",
+ " chosen_from_falcon, _, _ = get_chosen_and_not_chosen(picker)\n",
+ "\n",
+ " record = rg.FeedbackRecord(\n",
+ " fields={\n",
+ " \"instruction\": r[\"prompt\"],\n",
+ " chosen: r[\"original_response\"],\n",
+ " not_chosen: r[chosen_from_falcon],\n",
+ " },\n",
+ " responses=[{\"values\": {\"choose-best\": {\"value\": chosen_id + 1}}}],\n",
+ " external_id=r[\"external_id\"],\n",
+ " )\n",
+ " records.append(record)\n",
+ "\n",
+ "# create dataset\n",
+ "dataset = rg.FeedbackDataset(fields=fields, questions=[question], guidelines=guidelines)\n",
+ "\n",
+ "# add records and publish\n",
+ "dataset.add_records(records)\n",
+ "\n",
+ "dataset.push_to_huggingface(\"argilla/comparison-data-falcon-with-feedback\")"
+ ]
+ }
+ ],
+ "metadata": {
+ "accelerator": "GPU",
+ "colab": {
+ "gpuType": "T4",
+ "provenance": []
+ },
+ "kernelspec": {
+ "display_name": "Python 3",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.8.17"
+ },
+ "vscode": {
+ "interpreter": {
+ "hash": "2d98cb9bf90a932b5bf8e86e91214497eb0e38eb318595fbd6fbd5460fe92036"
+ }
+ },
+ "widgets": {
+ "application/vnd.jupyter.widget-state+json": {
+ "0432b133dc614d1fa8e2d06edc413302": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "### Enable Telemetry\n",
- "\n",
- "We gain valuable insights from how you interact with our tutorials. To improve ourselves in offering you the most suitable content, using the following lines of code will help us understand that this tutorial is serving you effectively. Though this is entirely anonymous, you can choose to skip this step if you prefer. For more info, please check out the [Telemetry](../../reference/telemetry.md) page."
- ]
+ "04c0309dc09e46419995addb7238f5ee": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "try:\n",
- " from argilla.utils.telemetry import tutorial_running\n",
- " tutorial_running()\n",
- "except ImportError:\n",
- " print(\"Telemetry is introduced in Argilla 1.20.0 and not found in the current installation. Skipping telemetry.\")"
- ]
+ "0e746ea7187c4b7382de0e5094d6234a": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "wmT7RQqlMBYW"
- },
- "source": [
- "## Configure the dataset\n",
- "As a first step, let's load the dataset and quickly explore the data. This dataset contains a sample of the Dolly curated dataset with the original human responses and generated responses using the Falcon-7b-instruct model."
- ]
+ "17ed61c237304d2bb77b80bd4def6466": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": "20px"
+ }
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "tYpQbwORMBYX",
- "outputId": "07f3808f-5965-4cb9-9375-0d3e92c2436d"
- },
- "outputs": [],
- "source": [
- "hf_dataset = load_dataset(\"argilla/dolly-curated-comparison-falcon-7b-instruct\", split=\"train\")"
- ]
+ "1c28d5ed9c7a41ad87e7fd83380e05a5": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
},
- {
- "cell_type": "code",
- "execution_count": 8,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 780
- },
- "id": "61DMAbebMBYY",
- "outputId": "97009132-c55e-4b64-9dc3-bb922b74e414"
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- "
\n",
- "\n",
- "
\n",
- " \n",
- "
\n",
- "
\n",
- "
prompt
\n",
- "
response-1
\n",
- "
response-2
\n",
- "
category
\n",
- "
original_response
\n",
- "
external_id
\n",
- "
\n",
- " \n",
- " \n",
- "
\n",
- "
0
\n",
- "
What is Depreciation
\n",
- "
What is Depreciation – 10 Important Facts to K...
\n",
- "
What is Depreciation on a Car?\\nDepreciation i...
\n",
- "
open_qa
\n",
- "
Depreciation is the drop in value of an asset ...
\n",
- "
518
\n",
- "
\n",
- "
\n",
- "
1
\n",
- "
What do you know about the city of Aberdeen in...
\n",
- "
Aberdeen, the city in which I've lived for the...
\n",
- "
As an AI language model, I don't have personal...
\n",
- "
open_qa
\n",
- "
Aberdeen is a city located in the North East o...
\n",
- "
351
\n",
- "
\n",
- "
\n",
- "
2
\n",
- "
Describe thunderstorm season in the United Sta...
\n",
- "
Describe thunderstorm season in the United Sta...
\n",
- "
Describe thunderstorm season in the United Sta...
\n",
- "
information_extraction
\n",
- "
Thunderstorm season in the United States and C...
\n",
- "
10567
\n",
- "
\n",
- "
\n",
- "
3
\n",
- "
When did Peloton IPO?\\nOn September 26, 2019, ...
\n",
- "
When did Peloton IPO?\\nPeloton launched its in...
\n",
- "
When did Peloton IPO?\\nPeloton IPO'd on May 26...
\n",
- "
closed_qa
\n",
- "
Peloton became a public company via an initial...
\n",
- "
12412
\n",
- "
\n",
- "
\n",
- "
4
\n",
- "
What is the best way to answer an interview qu...
\n",
- "
The best way to answer an interview question m...
\n",
- "
Some of the best ways to answer an interview q...
\n",
- "
creative_writing
\n",
- "
The first recommended step is to ask clarifyin...
\n",
- "
2521
\n",
- "
\n",
- "
\n",
- "
...
\n",
- "
...
\n",
- "
...
\n",
- "
...
\n",
- "
...
\n",
- "
...
\n",
- "
...
\n",
- "
\n",
- "
\n",
- "
7396
\n",
- "
How do i accept the change
\n",
- "
How do i accept the change in my life\\nAccepti...
\n",
- "
I's a great opportunity to improve. The only t...
\n",
- "
brainstorming
\n",
- "
Embrace the change and see the difference
\n",
- "
15010
\n",
- "
\n",
- "
\n",
- "
7397
\n",
- "
Extract the teams that the footballer Sócrates...
\n",
- "
Extract the teams that the footballer Sócrates...
\n",
- "
Extract the teams that the footballer Sócrates...
\n",
- "
information_extraction
\n",
- "
Brazil, Botafogo-SP, Corinthians, Fiorentina
\n",
- "
9970
\n",
- "
\n",
- "
\n",
- "
7398
\n",
- "
Without quoting directly from the text give me...
\n",
- "
Without quoting directly from the text give me...
\n",
- "
Without quoting directly from the text give me...
\n",
- "
summarization
\n",
- "
Brendon Small is a stand-up comedian, Creator...
\n",
- "
14205
\n",
- "
\n",
- "
\n",
- "
7399
\n",
- "
Is Killing is Sin ? Is it ture
\n",
- "
Is Killing is Sin ? Is it ture?\\nKilling can b...
\n",
- "
Is Killing is Sin ? Is it ture?\\nKilling is no...
\n",
- "
brainstorming
\n",
- "
Killing a human being should not be sin becaus...
\n",
- "
11253
\n",
- "
\n",
- "
\n",
- "
7400
\n",
- "
Who was Otto von Bismarck?\\nOtto, Prince of Bi...
\n",
- "
Who was Otto von Bismarck?\\nOtto von Bismarck ...
\n",
- "
Who was Otto von Bismarck?\\nOtto von Bismarck ...
\n",
- "
information_extraction
\n",
- "
Otto von Bismarck was a Prussian and German so...
\n",
- "
12872
\n",
- "
\n",
- " \n",
- "
\n",
- "
7401 rows × 6 columns
\n",
- "
"
- ],
- "text/plain": [
- " prompt \\\n",
- "0 What is Depreciation \n",
- "1 What do you know about the city of Aberdeen in... \n",
- "2 Describe thunderstorm season in the United Sta... \n",
- "3 When did Peloton IPO?\\nOn September 26, 2019, ... \n",
- "4 What is the best way to answer an interview qu... \n",
- "... ... \n",
- "7396 How do i accept the change \n",
- "7397 Extract the teams that the footballer Sócrates... \n",
- "7398 Without quoting directly from the text give me... \n",
- "7399 Is Killing is Sin ? Is it ture \n",
- "7400 Who was Otto von Bismarck?\\nOtto, Prince of Bi... \n",
- "\n",
- " response-1 \\\n",
- "0 What is Depreciation – 10 Important Facts to K... \n",
- "1 Aberdeen, the city in which I've lived for the... \n",
- "2 Describe thunderstorm season in the United Sta... \n",
- "3 When did Peloton IPO?\\nPeloton launched its in... \n",
- "4 The best way to answer an interview question m... \n",
- "... ... \n",
- "7396 How do i accept the change in my life\\nAccepti... \n",
- "7397 Extract the teams that the footballer Sócrates... \n",
- "7398 Without quoting directly from the text give me... \n",
- "7399 Is Killing is Sin ? Is it ture?\\nKilling can b... \n",
- "7400 Who was Otto von Bismarck?\\nOtto von Bismarck ... \n",
- "\n",
- " response-2 \\\n",
- "0 What is Depreciation on a Car?\\nDepreciation i... \n",
- "1 As an AI language model, I don't have personal... \n",
- "2 Describe thunderstorm season in the United Sta... \n",
- "3 When did Peloton IPO?\\nPeloton IPO'd on May 26... \n",
- "4 Some of the best ways to answer an interview q... \n",
- "... ... \n",
- "7396 I's a great opportunity to improve. The only t... \n",
- "7397 Extract the teams that the footballer Sócrates... \n",
- "7398 Without quoting directly from the text give me... \n",
- "7399 Is Killing is Sin ? Is it ture?\\nKilling is no... \n",
- "7400 Who was Otto von Bismarck?\\nOtto von Bismarck ... \n",
- "\n",
- " category \\\n",
- "0 open_qa \n",
- "1 open_qa \n",
- "2 information_extraction \n",
- "3 closed_qa \n",
- "4 creative_writing \n",
- "... ... \n",
- "7396 brainstorming \n",
- "7397 information_extraction \n",
- "7398 summarization \n",
- "7399 brainstorming \n",
- "7400 information_extraction \n",
- "\n",
- " original_response external_id \n",
- "0 Depreciation is the drop in value of an asset ... 518 \n",
- "1 Aberdeen is a city located in the North East o... 351 \n",
- "2 Thunderstorm season in the United States and C... 10567 \n",
- "3 Peloton became a public company via an initial... 12412 \n",
- "4 The first recommended step is to ask clarifyin... 2521 \n",
- "... ... ... \n",
- "7396 Embrace the change and see the difference 15010 \n",
- "7397 Brazil, Botafogo-SP, Corinthians, Fiorentina 9970 \n",
- "7398 Brendon Small is a stand-up comedian, Creator... 14205 \n",
- "7399 Killing a human being should not be sin becaus... 11253 \n",
- "7400 Otto von Bismarck was a Prussian and German so... 12872 \n",
- "\n",
- "[7401 rows x 6 columns]"
- ]
- },
- "execution_count": 8,
- "metadata": {},
- "output_type": "execute_result"
- }
+ "1eca0bb4151543098cfbb98b64191f92": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_299f4ae7c6524b9d8a06f22063d5a92f",
+ "IPY_MODEL_4ed8d36201904c6cb43d0fcdf7b18f34",
+ "IPY_MODEL_b9ed9f3bcbfd4403b4c1ea9eb3001543"
],
- "source": [
- "df = hf_dataset.to_pandas()\n",
- "df"
- ]
+ "layout": "IPY_MODEL_a45c853e3a7c4b65ad3bdae9a67607bc"
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "-OMxM53bMBYZ"
- },
- "source": [
- "For reward modeling, we would like to ask labelers to rank two responses for a prompt from best to worst. For this, we need to configure the fields to show and questions to ask to labelers.\n",
- "\n",
- "The dataset will show the users three fields `instruction`, which corresponds to the prompt, `response-1` and `response-2`, which correspond to each of the responses to the instruction.\n"
- ]
+ "23cccf0f1f2240ecaabb5d6efa7a8bbb": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
},
- {
- "cell_type": "code",
- "execution_count": 9,
- "metadata": {
- "id": "VZ5prVQNMBYa"
- },
- "outputs": [],
- "source": [
- "# list of fields that we will use later for our dataset settings\n",
- "fields = [\n",
- " rg.TextField(name=\"instruction\", title=\"User instruction\"),\n",
- " rg.TextField(name=\"response-1\"),\n",
- " rg.TextField(name=\"response-2\")\n",
- "]"
- ]
+ "23ef7fb2393f45d885d11ada8e5a3534": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "VKU888AnMBYa"
- },
- "source": [
- "In this use case, we will ask the users to select the best response. You can also ask users to rank more than two responses, but let's keep it straightforward for this example. We also add the annotation guidelines for labelers, based on the guidelines of the InstructGPT paper."
- ]
+ "254c7bb9621c457182976287a0a5ac66": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
},
- {
- "cell_type": "code",
- "execution_count": 10,
- "metadata": {
- "id": "wJvW18MEMBYb"
- },
- "outputs": [],
- "source": [
- "question = rg.RatingQuestion(\n",
- " name=\"choose-best\",\n",
- " title=\"Choose the best response:\",\n",
- " description=\"Choose the most helpful, harmless, and truthful response. Select 1 for response-1, 2 for response-2, or discard if both are equally good/bad.\",\n",
- " values=[1,2],\n",
- " required=True\n",
- ")\n",
- "\n",
- "guidelines=\"\"\"These guidelines are based on the paper [Training Language Models to Follow Instructions with Human Feedback](https://cdn.openai.com/papers/Training_language_models_to_follow_instructions_with_human_feedback.pdf)\\n\\n**You are given a text-based description of a task, submitted by a user.**\\n\\nThis task description may be in the form of an explicit instruction (e.g. \\\"Write a story about a wise frog.\\\"). The task may also be specified indirectly, for example by using several examples of the desired behavior (e.g. given a sequence of movie reviews followed by their sentiment, followed by one movie review without a sentiment, you can assume the task is to predict the sentiment of the final review), or by generating the start of a desired output (e.g. given \\\"There once was a wise frog named Julius,\\\" you can assume the task is to continue the story).\\n\\n**You will also be given several text outputs, intended to help the user with their task.** Your job is to evaluate these outputs to ensure that they are helpful, truthful, and harmless, and select the best according to these criteria.\\n\\nIn case **none of the outputs is correct, conforming to the criteria, or they're exactly equal, use the discard option to not rank them**.\\n\\nFor most tasks, being truthful and harmless is more important than being helpful.\\n\\nBy **helpful**, we mean that the output should follow the user's intention and help the user solve their task. Some examples of helpful behavior:\\n\\n- Writing in clear language.\\n- Answering the question they meant to ask, even if they mis-asked it.\\n- Being sensitive to internationality (e.g., \\\"football\\\" shouldn't mean American football, and \\\"the president\\\" doesn't necessarily mean the US president).\\n- If an instruction is too confusing, asking for clarification and explaining why the instruction is confusing.\\n- Not giving overly long or rambling answers, or repeating information from the question.\\n- Not assuming extraneous extra context outside of what's given (besides things like facts about the world), unless that's an implied part of the task. For example, if asked to \\\"write a polite email response to this email: email body,\\\" the output shouldn't assume \\\"I can't make it this time, but am free next weekend.\\\" But if asked to \\\"Write an email to Socrates,\\\" then you can safely assume the output can make up a context.\\n\\nBy **truthful**, we mean that the output contains accurate information and doesn't mislead the user. Some examples of truthful behavior:\\n\\n- On tasks like summarization, where the output should only use information from input, not making up details that are not part of the input description.\\n- Not producing clearly false information about the world (e.g., making up facts or promoting conspiracies). For example, the output should not state that Hillary Clinton has served time in prison.\\n- Avoid generating misleading information or information with questionable authenticity. For example, if the input asks \\\"Why did Hillary Clinton go to jail?\\\", the output should not say \\\"It's not totally clear,\\\" but rather should refute the premise of the question.\\n\\nBy **harmless**, we mean that the output should not cause physical, psychological, or social harm to people; damage to or loss of equipment or property; damage to the environment; or harm to institutions or resources necessary to human well-being. Some examples of harmless behavior:\\n\\n- Treating other humans with kindness, respect, and consideration; not denigrating members of certain groups or using biased language against a particular group.\\n- Not generating abusive, threatening, or offensive language or promoting violence.\\n- Not writing sexual or violent content if it's not asked for.\\n- Not giving bad real-world advice or promoting illegal activity.\\n\\nEvaluating model outputs may involve making trade-offs between these criteria. These trade-offs will depend on the task.\"\"\""
- ]
+ "299f4ae7c6524b9d8a06f22063d5a92f": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_fc422a04dbcf4b979fa7569c171ac971",
+ "placeholder": "",
+ "style": "IPY_MODEL_a291ef8013ee445b8f3e36be30e1da69",
+ "value": "Downloading data: 100%"
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "RfZuuRpr07n4"
- },
- "source": [
- "## Create records\n",
- "\n",
- "The next step is to build the records for collecting comparison data. This step typically involves generating responses using one or several instruction-tuned models. \n",
- "\n",
- ".. tip:: When showing responses from two different models to labelers it's recommended to randomly assign different model responses to `response-1` and `response-2` for each record. Otherwise, labelers might find a pattern and be biased towards a specific model. This is especially relevant for model comparison and evaluation but also applies to comparison data for reward modeling. \n",
- "\n",
- "In this example, we've already generated a dataset using the instructions from the Dolly curated dataset with the Falcon-7B-instruct model. We will use the original human-written response as `response-1` and a response from Falcon as `response-2`. \n",
- "\n",
- "You can build the records and publish them for labelers as follows:"
- ]
+ "2a9d2460cf024776bb0010ff6f1a5fa7": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_6e00a979fa604747b320018d4d5549ad",
+ "max": 7401,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_5ed78905729747d8a3dfe68590950d73",
+ "value": 7401
+ }
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "qZUeAT-1OFfz",
- "outputId": "2b605fd6-3708-4f8c-d18a-4dc1f75f981e"
- },
- "outputs": [],
- "source": [
- "# build records from hf dataset\n",
- "records = [\n",
- " rg.FeedbackRecord(fields={\"instruction\": r[\"prompt\"], \"response-1\": r[\"original_response\"], \"response-2\": r[\"response-2\"]})\n",
- " for r in hf_dataset\n",
- "]\n",
- "\n",
- "# create dataset\n",
- "dataset = rg.FeedbackDataset(\n",
- " fields=fields,\n",
- " questions=[question],\n",
- " guidelines=guidelines\n",
- ")\n",
- "\n",
- "# add records and publish\n",
- "dataset.add_records(records)\n",
- "dataset.push_to_argilla(\"comparison-data-falcon\", workspace=\"admin\")\n"
- ]
+ "2b34ee7a285a4dbb8c1001d8162b8e44": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "Now the dataset is ready for labeling. This is the Feedback UI we have just configured:\n",
- "\n",
- "\n",
- "\n",
- "\n"
- ]
+ "324c48fb64fb4021a4acec428fcf53ac": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "4vUhGWWnufDq"
- },
- "source": [
- "Additionally, you can push the dataset to the Hub for reproducibility and reuse. This dataset is available in the Hub, feel free to [read the dataset card](https://huggingface.co/datasets/argilla/comparison-data-falcon-with-feedback) to understand its structure, annotation guidelines, and how to import it."
- ]
+ "3aaa39cccc26416f9c7aeab4d50da8c5": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "id": "0kqc8fADoX4Q"
- },
- "outputs": [],
- "source": [
- "#dataset.push_to_huggingface(\"comparison-data-falcon\")"
- ]
+ "3cc6da03d28447d0a7259c460b017296": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_ffc2b422e5284a3180e703cab2ebef1e",
+ "IPY_MODEL_5151b3406f774c57b66c0651e10edb42",
+ "IPY_MODEL_b04603b9ced743d991a61e42fc0d8ce6"
+ ],
+ "layout": "IPY_MODEL_99f75e3565334a0cad80b0ddf434156c"
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "aW40aC5xvhrR"
- },
- "source": [
- "## Collect feedback and prepare the dataset\n",
- "\n",
- "Once the data has been labeled using the Argilla UI, we can retrieve it with the Python SDK and prepare it for training the reward model with TRL.\n",
- "\n",
- "If you are running this tutorial but haven´t labeled any data points, execute the following cell to retrieve the labeled dataset from the Hugging Face Hub. This dataset already contains ranked responses and can be used for the next steps. The dataset is available in the Hub, feel free to [read the dataset card](https://huggingface.co/datasets/argilla/comparison-data-falcon-with-feedback) to understand its structure, annotation guidelines, responses, and how to import it. \n",
- "\n",
- "If you have labeled some examples jump and execute the second cell. "
- ]
+ "4d451a340eb341ee88002e9ecce1d2a9": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 238,
- "referenced_widgets": [
- "b4a7efcf448247c586546cee7ae4e853",
- "ad4dd7f68c00437e878e2965e08457c8",
- "e7b96b86f6df4826b04171eaec7fa2a6",
- "56021dd10ea447029b2d64a3954f50a8",
- "ff6274caa2cb4346b9862dcbf1846184",
- "a681213d4f1f4c0db8cf2be709734067",
- "626d87ba234a4af9aed53f633cefbd95",
- "1c28d5ed9c7a41ad87e7fd83380e05a5",
- "dab35d7560824b828e1976691e481301",
- "23cccf0f1f2240ecaabb5d6efa7a8bbb",
- "f5d4cf0d204f478094976a30309de13e",
- "1eca0bb4151543098cfbb98b64191f92",
- "299f4ae7c6524b9d8a06f22063d5a92f",
- "4ed8d36201904c6cb43d0fcdf7b18f34",
- "b9ed9f3bcbfd4403b4c1ea9eb3001543",
- "a45c853e3a7c4b65ad3bdae9a67607bc",
- "fc422a04dbcf4b979fa7569c171ac971",
- "a291ef8013ee445b8f3e36be30e1da69",
- "7ba0eca5e7984391b4d32608c85bd236",
- "0e746ea7187c4b7382de0e5094d6234a",
- "324c48fb64fb4021a4acec428fcf53ac",
- "841a58e70b3f4621a4419d2872b3d348",
- "3cc6da03d28447d0a7259c460b017296",
- "ffc2b422e5284a3180e703cab2ebef1e",
- "5151b3406f774c57b66c0651e10edb42",
- "b04603b9ced743d991a61e42fc0d8ce6",
- "99f75e3565334a0cad80b0ddf434156c",
- "e295f279b61f48d683784131a8d533cc",
- "254c7bb9621c457182976287a0a5ac66",
- "9e60f5105a2f430d9c5f382c54b0610c",
- "2b34ee7a285a4dbb8c1001d8162b8e44",
- "4d451a340eb341ee88002e9ecce1d2a9",
- "8c1c4cac8f0444e4a69bd316efde9080",
- "b3dff83de3894147838358514c8ef63c",
- "ac4deaed71894ca9b8bdba7e14a34212",
- "8ed4b17c0dc4464cb5cc8bcfbc6bc803",
- "9a45be3cbddc4d0cbcc3591fe2c2883e",
- "51d0e514f6244692b2b1e6dbfcc0a460",
- "0432b133dc614d1fa8e2d06edc413302",
- "ffe071addb80421dba36b5797a594dbe",
- "17ed61c237304d2bb77b80bd4def6466",
- "809240690fb44738b6173cc53504bcf1",
- "5d3a83515f8944d491fe3305ee01a314",
- "52d358e242dd4876a66adbdb89fb03fc",
- "dd782a3398d1437f9d3626bdf4b87be4",
- "f632551058f744b0b9ad9696397c8d53",
- "914b7ab6f21d486fa42be37545ea73d7",
- "94903b2e56ef43b9b39ddd576a5a06ee",
- "902a2c7722484d4283e5ba0757641a0e",
- "23ef7fb2393f45d885d11ada8e5a3534",
- "3aaa39cccc26416f9c7aeab4d50da8c5",
- "04c0309dc09e46419995addb7238f5ee",
- "a9dcbe78ae6641d38775818748dd71c4",
- "5c23c54cc6bd4475ba755490afb520db",
- "bda482b92a8d4229983e72581154eba9"
- ]
- },
- "id": "p-0_iwa8IA2y",
- "outputId": "abbc3e3a-6053-4a0e-c1a1-f3f5f32d686c"
- },
- "outputs": [],
- "source": [
- "# if you haven't ranked any responses with the UI run this cell\n",
- "# otherwise ran the next one\n",
- "feedback_dataset = rg.FeedbackDataset.from_huggingface(\"argilla/comparison-data-falcon-with-feedback\")"
- ]
+ "4ed8d36201904c6cb43d0fcdf7b18f34": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_7ba0eca5e7984391b4d32608c85bd236",
+ "max": 5211692,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_0e746ea7187c4b7382de0e5094d6234a",
+ "value": 5211692
+ }
},
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/"
- },
- "id": "TZGfBzgmvg1d",
- "outputId": "7efee8d2-79b2-4555-a489-f8a55fcbec31"
- },
- "outputs": [],
- "source": [
- "# run this cell if you have ranked the responses in the UI\n",
- "feedback_dataset = rg.FeedbackDataset.from_argilla('comparison-data-falcon', workspace=\"admin\")"
- ]
+ "5151b3406f774c57b66c0651e10edb42": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_9e60f5105a2f430d9c5f382c54b0610c",
+ "max": 1,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_2b34ee7a285a4dbb8c1001d8162b8e44",
+ "value": 1
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "The next step is to prepare the dataset in the standard format for training a reward model. In particular, we want to select the chosen and rejected responses from the user feedback. We do this by creating a `TrainingTask` instance for reward modeling using a function that returns chosen-rejected tuples."
- ]
+ "51d0e514f6244692b2b1e6dbfcc0a460": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": "hidden",
+ "width": null
+ }
},
- {
- "cell_type": "code",
- "execution_count": 3,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 537
- },
- "id": "cll7j10swNT2",
- "outputId": "0e4974e6-74b1-41be-c54c-92bca9732ddc"
- },
- "outputs": [],
- "source": [
- "from typing import Any, Dict\n",
- "from argilla.feedback import TrainingTask\n",
- "from collections import Counter\n",
- "\n",
- "def formatting_func(sample: Dict[str, Any]):\n",
- " # sample[\"choose-best\"] => [{'user_id': None, 'value': 1, 'status': 'submitted'}, ...]\n",
- " values = [\n",
- " annotation[\"value\"]\n",
- " for annotation in sample[\"choose-best\"]\n",
- " if annotation[\"status\"] == \"submitted\"\n",
- " ]\n",
- " \n",
- " # We will only focus on the annotated records in the dataset\n",
- " if Counter(values).most_common(1) != []:\n",
- " \n",
- " # values => [1]\n",
- " winning_response = Counter(values).most_common(1)[0][0]\n",
- " print(Counter(values).most_common(1))\n",
- " if winning_response == 1:\n",
- " chosen = sample[\"response-1\"]\n",
- " rejected = sample[\"response-2\"]\n",
- " else:\n",
- " chosen = sample[\"response-2\"]\n",
- " rejected = sample[\"response-1\"]\n",
- " return chosen, rejected\n",
- "\n",
- "task = TrainingTask.for_reward_modeling(formatting_func=formatting_func)"
- ]
+ "52d358e242dd4876a66adbdb89fb03fc": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
},
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "If we want, we can observe the resulting dataset by preparing it for training with TRL:"
- ]
+ "56021dd10ea447029b2d64a3954f50a8": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_23cccf0f1f2240ecaabb5d6efa7a8bbb",
+ "placeholder": "",
+ "style": "IPY_MODEL_f5d4cf0d204f478094976a30309de13e",
+ "value": " 1/1 [00:04<00:00, 4.35s/it]"
+ }
},
- {
- "cell_type": "code",
- "execution_count": 4,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "Dataset({\n",
- " features: ['chosen', 'rejected'],\n",
- " num_rows: 7401\n",
- "})"
- ]
- },
- "execution_count": 4,
- "metadata": {},
- "output_type": "execute_result"
- }
+ "5c23c54cc6bd4475ba755490afb520db": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "5d3a83515f8944d491fe3305ee01a314": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "5ed78905729747d8a3dfe68590950d73": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "626d87ba234a4af9aed53f633cefbd95": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "660b25b7377a4bd0a2baf03a437dc060": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "6e00a979fa604747b320018d4d5549ad": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "7ba0eca5e7984391b4d32608c85bd236": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "7d01e2401fa44199afb31d5dcc443761": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "809240690fb44738b6173cc53504bcf1": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "841a58e70b3f4621a4419d2872b3d348": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "8a6d78da941d429b981d82a1b0d249ab": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_fdd275ff03504203b5dbce2ea732fd44",
+ "placeholder": "",
+ "style": "IPY_MODEL_7d01e2401fa44199afb31d5dcc443761",
+ "value": " 7378/7401 [00:23<00:00, 318.13 examples/s]"
+ }
+ },
+ "8b2770c23c424f8698d7f3ca60bf5d9a": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": "hidden",
+ "width": null
+ }
+ },
+ "8c1c4cac8f0444e4a69bd316efde9080": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "8ed4b17c0dc4464cb5cc8bcfbc6bc803": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "info",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_17ed61c237304d2bb77b80bd4def6466",
+ "max": 1,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_809240690fb44738b6173cc53504bcf1",
+ "value": 1
+ }
+ },
+ "902a2c7722484d4283e5ba0757641a0e": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "914b7ab6f21d486fa42be37545ea73d7": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "FloatProgressModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "FloatProgressModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "ProgressView",
+ "bar_style": "success",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_04c0309dc09e46419995addb7238f5ee",
+ "max": 1,
+ "min": 0,
+ "orientation": "horizontal",
+ "style": "IPY_MODEL_a9dcbe78ae6641d38775818748dd71c4",
+ "value": 1
+ }
+ },
+ "94903b2e56ef43b9b39ddd576a5a06ee": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_5c23c54cc6bd4475ba755490afb520db",
+ "placeholder": "",
+ "style": "IPY_MODEL_bda482b92a8d4229983e72581154eba9",
+ "value": " 1/1 [00:00<00:00, 17.55it/s]"
+ }
+ },
+ "99f75e3565334a0cad80b0ddf434156c": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "9a45be3cbddc4d0cbcc3591fe2c2883e": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_5d3a83515f8944d491fe3305ee01a314",
+ "placeholder": "",
+ "style": "IPY_MODEL_52d358e242dd4876a66adbdb89fb03fc",
+ "value": " 7401/0 [00:00<00:00, 62827.72 examples/s]"
+ }
+ },
+ "9a872753cf9a4fde9caa69660e85c896": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_9fc9251942db44548dcc9b55312f6963",
+ "placeholder": "",
+ "style": "IPY_MODEL_660b25b7377a4bd0a2baf03a437dc060",
+ "value": "Map: 100%"
+ }
+ },
+ "9e60f5105a2f430d9c5f382c54b0610c": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "9fc9251942db44548dcc9b55312f6963": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "a291ef8013ee445b8f3e36be30e1da69": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "DescriptionStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "DescriptionStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "description_width": ""
+ }
+ },
+ "a45c853e3a7c4b65ad3bdae9a67607bc": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "a681213d4f1f4c0db8cf2be709734067": {
+ "model_module": "@jupyter-widgets/base",
+ "model_module_version": "1.2.0",
+ "model_name": "LayoutModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/base",
+ "_model_module_version": "1.2.0",
+ "_model_name": "LayoutModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "LayoutView",
+ "align_content": null,
+ "align_items": null,
+ "align_self": null,
+ "border": null,
+ "bottom": null,
+ "display": null,
+ "flex": null,
+ "flex_flow": null,
+ "grid_area": null,
+ "grid_auto_columns": null,
+ "grid_auto_flow": null,
+ "grid_auto_rows": null,
+ "grid_column": null,
+ "grid_gap": null,
+ "grid_row": null,
+ "grid_template_areas": null,
+ "grid_template_columns": null,
+ "grid_template_rows": null,
+ "height": null,
+ "justify_content": null,
+ "justify_items": null,
+ "left": null,
+ "margin": null,
+ "max_height": null,
+ "max_width": null,
+ "min_height": null,
+ "min_width": null,
+ "object_fit": null,
+ "object_position": null,
+ "order": null,
+ "overflow": null,
+ "overflow_x": null,
+ "overflow_y": null,
+ "padding": null,
+ "right": null,
+ "top": null,
+ "visibility": null,
+ "width": null
+ }
+ },
+ "a9dcbe78ae6641d38775818748dd71c4": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "ProgressStyleModel",
+ "state": {
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "ProgressStyleModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/base",
+ "_view_module_version": "1.2.0",
+ "_view_name": "StyleView",
+ "bar_color": null,
+ "description_width": ""
+ }
+ },
+ "ac4deaed71894ca9b8bdba7e14a34212": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_0432b133dc614d1fa8e2d06edc413302",
+ "placeholder": "",
+ "style": "IPY_MODEL_ffe071addb80421dba36b5797a594dbe",
+ "value": "Generating train split: "
+ }
+ },
+ "ad4dd7f68c00437e878e2965e08457c8": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_a681213d4f1f4c0db8cf2be709734067",
+ "placeholder": "",
+ "style": "IPY_MODEL_626d87ba234a4af9aed53f633cefbd95",
+ "value": "Downloading data files: 100%"
+ }
+ },
+ "b04603b9ced743d991a61e42fc0d8ce6": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_4d451a340eb341ee88002e9ecce1d2a9",
+ "placeholder": "",
+ "style": "IPY_MODEL_8c1c4cac8f0444e4a69bd316efde9080",
+ "value": " 1/1 [00:00<00:00, 26.03it/s]"
+ }
+ },
+ "b3dff83de3894147838358514c8ef63c": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_ac4deaed71894ca9b8bdba7e14a34212",
+ "IPY_MODEL_8ed4b17c0dc4464cb5cc8bcfbc6bc803",
+ "IPY_MODEL_9a45be3cbddc4d0cbcc3591fe2c2883e"
],
- "source": [
- "dataset = feedback_dataset.prepare_for_training(framework=\"trl\", task=task)\n",
- "dataset"
- ]
+ "layout": "IPY_MODEL_51d0e514f6244692b2b1e6dbfcc0a460"
+ }
},
- {
- "cell_type": "code",
- "execution_count": 5,
- "metadata": {},
- "outputs": [
- {
- "data": {
- "text/plain": [
- "{'chosen': \"Depreciation is the drop in value of an asset due to wear and tear, age and obsolescence (going out of date) as recorded in an organization's financial records.\",\n",
- " 'rejected': 'What is Depreciation – 10 Important Facts to Know?\\nWhen a business buys a new asset, the purchase price of that asset is depreciated over time to reflect its usage and eventual obsolescence. Depreciation expense can be a tax deductible expense and is usually a non-cash expense reported on a company’s income statement and balance sheet. The amount of depreciation expense a company reports each year is the difference between the original purchase price of the asset and what the current value of that asset might be. Here are 10 important facts to know about depreciation:\\n1. Depreciation is a non-cash expense. It is an expense that is reported in a business’s income statement and balance sheet and not a cash flow expense.\\n2. Depreciation is an accounting standard and it is required to be disclosed in a business’s financial statements.\\n3. The amount of depreciation is usually a tax expense and not a cash expense reported on a company’s income statement'}"
- ]
- },
- "execution_count": 5,
- "metadata": {},
- "output_type": "execute_result"
- }
+ "b44d62c0bef242fabab85ff6129b42dc": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_9a872753cf9a4fde9caa69660e85c896",
+ "IPY_MODEL_2a9d2460cf024776bb0010ff6f1a5fa7",
+ "IPY_MODEL_8a6d78da941d429b981d82a1b0d249ab"
],
- "source": [
- "dataset[0]"
- ]
+ "layout": "IPY_MODEL_8b2770c23c424f8698d7f3ca60bf5d9a"
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "UgoIyJanMBYk"
- },
- "source": [
- "This dataset is ready to be used as comparison data to train a reward model.\n",
- "\n",
- "
\n",
- "\n",
- "Note\n",
- "\n",
- "The paper Direct Preference Optimization: Your Language Model is Secretly a Reward Model proposes DPO, a promising method for using comparison data directly to model human preference, eliminating the need for a reward model and the RL step. Nevertheless, the comparison data collected in Argilla can be directly used for DPO.\n",
- "\n",
- "
"
- ]
+ "b4a7efcf448247c586546cee7ae4e853": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HBoxModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HBoxModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HBoxView",
+ "box_style": "",
+ "children": [
+ "IPY_MODEL_ad4dd7f68c00437e878e2965e08457c8",
+ "IPY_MODEL_e7b96b86f6df4826b04171eaec7fa2a6",
+ "IPY_MODEL_56021dd10ea447029b2d64a3954f50a8"
+ ],
+ "layout": "IPY_MODEL_ff6274caa2cb4346b9862dcbf1846184"
+ }
},
- {
- "attachments": {},
- "cell_type": "markdown",
- "metadata": {
- "id": "F8r9_XJi1K9P"
- },
- "source": [
- "## Train the reward model with `trl`\n",
- "In this step, we will use the `RewardTrainer` from the trl library. To understand this step, we recommend you to [check the trl docs](https://huggingface.co/docs/trl/reward_trainer). \n",
- "\n",
- "
\n",
- "To run this step, you need to rank some examples using the Argilla UI, or run the step above with the load from Hugging Face call: `feedback_dataset = FeedbackDataset.from_huggingface`\n",
- "
\n",
- "\n",
- "To train a Reward Model, you need to choose a base model to fine-tune. In the literature, the base model is typically the supervised fine-tuned model resulting from the instruction-tuning step. In this example, that would mean using the [Falcon-7B-instruct model](https://huggingface.co/tiiuae/falcon-7b-instruct). However, as Reward Models are essentially classifiers you can use a more lightweight backbone model, for this example we will use `distilroberta-base` but feel free to experiment with other models.\n",
- "\n",
- "The code below fine-tunes a `SequenceClassification` model with our preference dataset. The most interesting part is the `formatting_func` function. This function combines instructions with chosen and rejected responses, creating two new strings. These strings are tokenized, becoming input for a reward model that learns to distinguish between good and bad responses based on these examples. The model will be optimized to assign higher values to preferred responses and lower values to rejected responses.\n"
- ]
+ "b9ed9f3bcbfd4403b4c1ea9eb3001543": {
+ "model_module": "@jupyter-widgets/controls",
+ "model_module_version": "1.5.0",
+ "model_name": "HTMLModel",
+ "state": {
+ "_dom_classes": [],
+ "_model_module": "@jupyter-widgets/controls",
+ "_model_module_version": "1.5.0",
+ "_model_name": "HTMLModel",
+ "_view_count": null,
+ "_view_module": "@jupyter-widgets/controls",
+ "_view_module_version": "1.5.0",
+ "_view_name": "HTMLView",
+ "description": "",
+ "description_tooltip": null,
+ "layout": "IPY_MODEL_324c48fb64fb4021a4acec428fcf53ac",
+ "placeholder": "",
+ "style": "IPY_MODEL_841a58e70b3f4621a4419d2872b3d348",
+ "value": " 5.21M/5.21M [00:01<00:00, 5.76MB/s]"
+ }
},
- {
- "cell_type": "code",
- "execution_count": 7,
- "metadata": {
- "colab": {
- "base_uri": "https://localhost:8080/",
- "height": 516,
- "referenced_widgets": [
- "b44d62c0bef242fabab85ff6129b42dc",
- "9a872753cf9a4fde9caa69660e85c896",
- "2a9d2460cf024776bb0010ff6f1a5fa7",
- "8a6d78da941d429b981d82a1b0d249ab",
- "8b2770c23c424f8698d7f3ca60bf5d9a",
- "9fc9251942db44548dcc9b55312f6963",
- "660b25b7377a4bd0a2baf03a437dc060",
- "6e00a979fa604747b320018d4d5549ad",
- "5ed78905729747d8a3dfe68590950d73",
- "fdd275ff03504203b5dbce2ea732fd44",
- "7d01e2401fa44199afb31d5dcc443761"
- ]
- },
- "id": "-MtSUam61J_Z",
- "outputId": "6680758c-3b9c-444a-e73b-48bbb4c1b4b5"
- },
- "outputs": [
- {
- "data": {
- "text/html": [
- "