tok2vec loss doesnt change from 0 while training #13776
Unanswered
ravi-abv
asked this question in
Help: Model Advice
Replies: 0 comments
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Uh oh!
There was an error while loading. Please reload this page.
-
I am trying to predict categories of text of reviews using a pipeline of ['tok2vec', 'ner', 'textcat_multilabel']
The NER generates entities from the text which are then used in a multilabel text classifier to predict the categories
The tok2vec loss is always 0. How to ensure that tok2vec component gets trained
============================= Training pipeline =============================
ℹ Pipeline: ['tok2vec', 'ner', 'textcat_multilabel']
ℹ Initial learn rate: 0.001
E # LOSS TOK2VEC LOSS NER LOSS TEXTC... ENTS_F ENTS_P ENTS_R CATS_SCORE SCORE
0 0 0.00 168.67 0.28 0.05 0.10 0.03 19.34 0.10
0 200 0.00 6749.35 2.01 85.21 92.53 78.96 17.51 0.51
0 400 0.00 1756.33 0.76 88.96 91.53 86.53 20.18 0.55
0 600 0.00 1456.59 0.62 91.07 93.74 88.54 22.27 0.57
My config is as follows
[paths]
train = "corpus/train_data.spacy"
dev = "corpus/dev_data.spacy"
output = "training"
vectors = null
init_tok2vec = null
[system]
gpu_allocator = "pytorch"
seed = 42
[nlp]
lang = "en"
pipeline = ["tok2vec", "ner", "textcat_multilabel"]
disabled = []
batch_size = 512
tokenizer = {"@Tokenizers": "spacy.Tokenizer.v1"}
before_creation = null
after_creation = null
after_pipeline_creation = null
[components]
[components.ner]
factory = "ner"
incorrect_spans_key = null
moves = null
scorer = {"@scorers": "spacy.ner_scorer.v1"}
update_with_oracle_cut_size = 100
[components.ner.model]
@architectures = "spacy.TransitionBasedParser.v2"
state_type = "ner"
extra_state_tokens = false
hidden_width = 64
maxout_pieces = 2
use_upper = true
nO = null
[components.textcat_multilabel]
factory = "textcat_multilabel"
scorer = {"@scorers": "spacy.textcat_multilabel_scorer.v2"}
threshold = 0.4
[components.textcat_multilabel.model]
@architectures = "spacy.TextCatEnsemble.v2"
nO = null
[components.textcat_multilabel.model.tok2vec]
@architectures = "spacy.Tok2Vec.v2"
[components.textcat_multilabel.model.tok2vec.embed]
@architectures = "spacy.MultiHashEmbed.v2"
width = 64
rows = [2000, 2000, 500, 1000, 500]
attrs = ["NORM", "LOWER", "PREFIX", "SUFFIX", "SHAPE"]
include_static_vectors = false
[components.textcat_multilabel.model.tok2vec.encode]
@architectures = "spacy.MaxoutWindowEncoder.v2"
width = 64
window_size = 1
maxout_pieces = 3
depth = 2
[components.textcat_multilabel.model.linear_model]
@architectures = "spacy.TextCatBOW.v3"
exclusive_classes = false
ngram_size = 1
no_output_layer = false
nO = null
[components.tok2vec]
factory = "tok2vec"
[components.tok2vec.model]
@architectures = "spacy.Tok2Vec.v2"
[components.tok2vec.model.embed]
@architectures = "spacy.MultiHashEmbed.v2"
width = 300
attrs = ["NORM", "PREFIX", "SUFFIX", "SHAPE"]
rows = [5000, 2500, 2500, 2500]
include_static_vectors = false
[components.tok2vec.model.encode]
@architectures = "spacy.MaxoutWindowEncoder.v2"
width = 300
depth = 4
window_size = 1
maxout_pieces = 3
[corpora]
[corpora.dev]
@readers = "spacy.Corpus.v1"
path = ${paths.dev}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[corpora.train]
@readers = "spacy.Corpus.v1"
path = ${paths.train}
max_length = 0
gold_preproc = false
limit = 0
augmenter = null
[training]
max_epochs = 10
dropout = 0.2
train_corpus = "corpora.train"
dev_corpus = "corpora.dev"
seed = 42
gpu_allocator = "pytorch"
patience = 1600
max_steps = 20000
eval_frequency = 200
accumulate_gradient = 1
frozen_components = []
annotating_components = []
before_to_disk = null
before_update = null
[training.batcher]
@batchers = "spacy.batch_by_words.v1"
size = 512
tolerance = 0.2
discard_oversize = false
get_length = null
[training.logger]
@Loggers = "spacy.ConsoleLogger.v1"
progress_bar = false
[training.optimizer]
@optimizers = "Adam.v1"
beta1 = 0.9
beta2 = 0.999
L2_is_weight_decay = true
L2 = 0.01
grad_clip = 1.0
use_averages = false
eps = 0.00000001
learn_rate = 0.001
[training.score_weights]
ents_f = 0.5
ents_p = 0.0
ents_r = 0.0
ents_per_type = null
cats_score = 0.5
cats_score_desc = null
cats_micro_p = null
cats_micro_r = null
cats_micro_f = null
cats_macro_p = null
cats_macro_r = null
cats_macro_f = null
cats_macro_auc = null
cats_f_per_type = null
[initialize]
vectors = null
init_tok2vec = null
vocab_data = null
lookups = null
before_init = null
after_init = null
[initialize.components]
[initialize.tokenizer]
Beta Was this translation helpful? Give feedback.
All reactions