feat: add imdb distributed demo for mindnlp Trainer API

Author: Tridu33

examples/parallel/bert_imdb_finetune/bert_imdb_finetune_cpu_mindnlp_trainer_npus_same.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python
+# coding: utf-8
+"""
+python bert_imdb_finetune_cpu_mindnlp_trainer.py
+bash bert_imdb_finetune_npu_mindnlp_trainer.sh
+"""
+
+def main():
+    import mindspore
+    from mindspore.dataset import transforms
+    from mindnlp.engine import Trainer
+    from mindnlp.dataset import load_dataset
+
+    imdb_ds = load_dataset('imdb', split=['train', 'test'])
+    imdb_train = imdb_ds['train']
+    imdb_test = imdb_ds['test']
+    imdb_train.get_dataset_size()
+    from mindnlp.transformers import AutoTokenizer
+    # tokenizer
+    tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')
+
+    def process_dataset(dataset, tokenizer, max_seq_len=256, batch_size=32, shuffle=False):
+        is_ascend = mindspore.get_context('device_target') == 'Ascend'
+        def tokenize(text):
+            if is_ascend:
+                tokenized = tokenizer(text, padding='max_length', truncation=True, max_length=max_seq_len)
+            else:
+                tokenized = tokenizer(text, truncation=True, max_length=max_seq_len)
+            return tokenized['input_ids'], tokenized['token_type_ids'], tokenized['attention_mask']
+
+        if shuffle:
+            dataset = dataset.shuffle(batch_size)
+
+        # map dataset
+        dataset = dataset.map(operations=[tokenize], input_columns="text", output_columns=['input_ids', 'token_type_ids', 'attention_mask'])
+        dataset = dataset.map(operations=transforms.TypeCast(mindspore.int32), input_columns="label", output_columns="labels")
+        # batch dataset
+        if is_ascend:
+            dataset = dataset.batch(batch_size)
+        else:
+            dataset = dataset.padded_batch(batch_size, pad_info={'input_ids': (None, tokenizer.pad_token_id),
+                                                                'token_type_ids': (None, 0),
+                                                                'attention_mask': (None, 0)})
+
+        return dataset
+
+    # split train dataset into train and valid datasets
+    imdb_train, imdb_val = imdb_train.split([0.7, 0.3])
+
+    dataset_train = process_dataset(imdb_train, tokenizer, shuffle=True)
+    dataset_val = process_dataset(imdb_val, tokenizer)
+    dataset_test = process_dataset(imdb_test, tokenizer)
+
+    next(dataset_train.create_tuple_iterator())
+
+    from mindnlp.transformers import AutoModelForSequenceClassification
+
+    # set bert config and define parameters for training
+    model = AutoModelForSequenceClassification.from_pretrained('bert-base-cased', num_labels=2)
+
+    from mindnlp.engine import TrainingArguments
+    
+    training_args = TrainingArguments(
+        output_dir="bert_imdb_finetune_cpu",
+        evaluation_strategy="epoch",
+        save_strategy="epoch",
+        logging_strategy="epoch",
+        load_best_model_at_end=True,
+        num_train_epochs=2.0,
+        learning_rate=2e-5
+    )
+    training_args = training_args.set_optimizer(name="adamw", beta1=0.8) # OptimizerNames.SGD
+
+    from mindnlp import evaluate
+    import numpy as np
+    metric = evaluate.load("accuracy")
+    def compute_metrics(eval_pred):
+        logits, labels = eval_pred
+        predictions = np.argmax(logits, axis=-1)
+        return metric.compute(predictions=predictions, references=labels)
+    
+    trainer = Trainer(
+        model=model,
+        args=training_args,
+        train_dataset=dataset_train,
+        eval_dataset=dataset_val,
+        compute_metrics=compute_metrics
+    )
+    print("Start training")
+    trainer.train()
+
+    print("Start checking the test set")
+    trainer.evaluate(dataset_test)
+
+if __name__ == '__main__':
+    main()
+

examples/parallel/bert_imdb_finetune/bert_imdb_finetune_npu_mindnlp_trainer.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+echo "=========================================="
+echo "Please run the script as: "
+echo "bash bert_imdb_finetune_npu_mindnlp_trainer.sh"
+echo "==========================================="
+
+EXEC_PATH=$(pwd)
+if [ ! -d "${EXEC_PATH}/data" ]; then
+    if [ ! -f "${EXEC_PATH}/emotion_detection.tar.gz" ]; then
+        wget wget https://baidu-nlp.bj.bcebos.com/emotion_detection-dataset-1.0.0.tar.gz -O emotion_detection.tar.gz
+    fi
+    tar xvf emotion_detection.tar.gz
+fi
+export DATA_PATH=${EXEC_PATH}/data/
+
+rm -rf bert_imdb_finetune_cpu_mindnlp_trainer_npus_same
+mkdir bert_imdb_finetune_cpu_mindnlp_trainer_npus_same
+echo "start training"
+
+export MULTI_NPU="true" 
+# unset MULTI_NPU
+msrun --worker_num=2 --local_worker_num=2 --master_port=8119 \
+--log_dir=bert_imdb_finetune_cpu_mindnlp_trainer_npus_same --join=True \
+--cluster_time_out=30 bert_imdb_finetune_cpu_mindnlp_trainer_npus_same.py