Ordnung in tests

use `vocabulary.from_wordlist`, since starting the following commit,
other methods will be obsolete.

Author: jindrahelcl

tests/alignment.ini

@@ -47,12 +47,12 @@ series=["source"]
 data=["tests/data/val.tc.en"]
 
 [encoder_vocabulary]
-class=vocabulary.from_dataset
-datasets=[<train_data>]
-series_ids=["source"]
-max_size=60
-save_file="tests/outputs/alignment/encoder_vocabulary.pickle"
-overwrite=True
+class=vocabulary.from_wordlist
+path="tests/data/encoder_vocab.tsv"
+
+[decoder_vocabulary]
+class=vocabulary.from_wordlist
+path="tests/data/decoder_vocab.tsv"
 
 [encoder]
 class=encoders.recurrent.SentenceEncoder
@@ -68,14 +68,6 @@ class=attention.Attention
 name="attention_sentence_encoder"
 encoder=<encoder>
 
-[decoder_vocabulary]
-class=vocabulary.from_dataset
-datasets=[<train_data>]
-series_ids=["target"]
-max_size=70
-save_file="tests/outputs/alignment/decoder_vocabulary.pickle"
-overwrite=True
-
 [decoder]
 class=decoders.decoder.Decoder
 name="bahdanau_decoder"

tests/bahdanau.ini

@@ -40,12 +40,8 @@ data=["tests/data/val.tc.en"]
 outputs=[("encoded", "tests/outputs/bahdanau/encoded"), ("debugtensors", "tests/outputs/bahdanau/debugtensors")]
 
 [encoder_vocabulary]
-class=vocabulary.from_dataset
-datasets=[<train_data>]
-series_ids=["source"]
-max_size=60
-save_file="tests/outputs/bahdanau/encoder_vocabulary.pickle"
-overwrite=True
+class=vocabulary.from_wordlist
+path="tests/data/encoder_vocab.tsv"
 
 [encoder]
 class=encoders.recurrent.SentenceEncoder
@@ -62,12 +58,8 @@ name="attention_sentence_encoder"
 encoder=<encoder>
 
 [decoder_vocabulary]
-class=vocabulary.from_dataset
-datasets=[<train_data>]
-series_ids=["target"]
-max_size=70
-save_file="tests/outputs/bahdanau/decoder_vocabulary.pickle"
-overwrite=True
+class=vocabulary.from_wordlist
+path="tests/data/decoder_vocab.tsv"
 
 [decoder]
 class=decoders.decoder.Decoder

tests/beamsearch.ini

@@ -35,7 +35,7 @@ data=["tests/data/val.tc.en", "tests/data/val.tc.de", (processors.helpers.prepro
 
 [encoder_vocabulary]
 class=vocabulary.from_wordlist
-path="tests/outputs/vocab/encoder_vocab.tsv"
+path="tests/data/encoder_vocab.tsv"
 
 [inpseq]
 class=model.sequence.EmbeddedSequence
@@ -56,7 +56,7 @@ dropout_keep_prob=0.9
 
 [decoder_vocabulary]
 class=vocabulary.from_wordlist
-path="tests/outputs/vocab/decoder_vocab.tsv"
+path="tests/data/decoder_vocab.tsv"
 
 [decoder]
 class=decoders.transformer.TransformerDecoder

tests/beamsearch_ensembles.ini

@@ -35,7 +35,7 @@ data=["tests/data/val.tc.en", "tests/data/val.tc.de", (processors.helpers.prepro
 
 [encoder_vocabulary]
 class=vocabulary.from_wordlist
-path="tests/outputs/vocab/encoder_vocab.tsv"
+path="tests/data/encoder_vocab.tsv"
 
 [inpseq]
 class=model.sequence.EmbeddedSequence
@@ -56,7 +56,7 @@ dropout_keep_prob=0.9
 
 [decoder_vocabulary]
 class=vocabulary.from_wordlist
-path="tests/outputs/vocab/decoder_vocab.tsv"
+path="tests/data/decoder_vocab.tsv"
 
 [decoder]
 class=decoders.transformer.TransformerDecoder

tests/bpe.ini

@@ -49,12 +49,8 @@ merge_file="tests/data/merges_100.bpe"
 class=processors.bpe.BPEPostprocessor
 
 [bpe_vocabulary]
-class=vocabulary.from_dataset
-datasets=[<train_data>]
-series_ids=["source_bpe", "target_bpe"]
-max_size=209
-save_file="tests/outputs/bpe_vocabulary.pickle"
-overwrite=True
+class=vocabulary.from_wordlist
+path="tests/data/bpe_vocab.tsv"
 
 [encoder_input]
 class=model.sequence.EmbeddedSequence

tests/captioning.ini

@@ -59,12 +59,8 @@ name="attention_sentence_encoder"
 encoder=<imagenet>
 
 [decoder_vocabulary]
-class=vocabulary.from_dataset
-datasets=[<train_data>]
-series_ids=["target"]
-max_size=70
-save_file="tests/outputs/captioning/decoder_vocabulary.pickle"
-overwrite=True
+class=vocabulary.from_wordlist
+path="tests/data/decoder_vocab.tsv"
 
 [decoder]
 class=decoders.decoder.Decoder

tests/classifier.ini

@@ -34,7 +34,7 @@ data=["tests/data/val.tc.en", "tests/data/val.words", (processors.helpers.prepro
 
 [encoder_vocabulary]
 class=vocabulary.from_wordlist
-path="tests/outputs/vocab/encoder_vocab.tsv"
+path="tests/data/encoder_vocab.tsv"
 
 [encoder_rnn]
 class=encoders.recurrent.SentenceEncoder

tests/ctc.ini

@@ -47,11 +47,10 @@ feature_type="mfcc"
 delta_order=2
 
 [decoder_vocabulary]
-class=vocabulary.from_dataset
-datasets=[<train_data>]
-series_ids=["target"]
-max_size=5
-
+class=vocabulary.from_wordlist
+path="tests/data/yesno/yesno.vocab"
+contains_header=False
+contains_frequencies=False
 
 [audio_encoder]
 class=encoders.raw_rnn_encoder.RawRNNEncoder

tests/data/bpe_vocab.tsv

@@ -0,0 +1,210 @@
+Word	Word counts
+<pad>	1
+<s>	1
+</s>	1
+<unk>	1
+t@@	1632
+wo	135
+y@@	313
+o@@	1501
+un@@	535
+g	378
+,	663
+W@@	122
+hi@@	381
+te	167
+m@@	1099
+al@@	407
+es	340
+ar@@	840
+e	1429
+u@@	1316
+si@@	506
+d@@	1358
+n@@	818
+e@@	1515
+ar	104
+an@@	379
+y	429
+b@@	1493
+s@@	1449
+h@@	910
+.	1939
+v@@	491
+er@@	926
+al	55
+en	1559
+in	969
+d	716
+at@@	217
+s	1079
+p@@	1257
+ing	1045
+a	1814
+g@@	1326
+i@@	1415
+t	1452
+l@@	1441
+st@@	618
+em	160
+li@@	276
+tt@@	270
+le	344
+ir@@	122
+l	324
+c@@	740
+in@@	500
+to	140
+wo@@	187
+la@@	340
+man	407
+r@@	883
+is	273
+and@@	190
+on	519
+der	326
+w@@	882
+do@@	178
+w	92
+at	181
+the	368
+re@@	452
+f@@	986
+th@@	260
+er	798
+mi@@	78
+en@@	577
+k@@	799
+el@@	463
+on@@	266
+lo@@	189
+n	200
+et	165
+with	212
+ge	245
+ur@@	145
+bo@@	220
+of	217
+j@@	192
+q@@	17
+ur	28
+we@@	153
+to@@	83
+p	142
+a@@	1716
+ck	169
+and	389
+sp@@	181
+or@@	277
+m	379
+ri@@	196
+ra@@	314
+or	158
+ss@@	163
+ro@@	546
+ch	255
+o	28
+ei@@	376
+r	50
+ge@@	158
+f	56
+c	23
+M@@	284
+ro	1
+ch@@	631
+k	160
+und	439
+u	72
+i	15
+J@@	145
+B@@	278
+ck@@	267
+z@@	501
+be@@	151
+x@@	45
+an	245
+A@@	110
+wei@@	94
+au@@	299
+G@@	185
+C@@	26
+D@@	49
+&@@	28
+;@@	20
+do	3
+T@@	123
+V	3
+h	64
+th	21
+wi@@	86
+di@@	82
+eh@@	311
+b	30
+-@@	117
+ir	41
+3@@	1
+0	1
+el	69
+S@@	416
+x	6
+ein@@	127
+F@@	179
+P@@	187
+R@@	74
+;	10
+E@@	69
+Z@@	48
+hr@@	198
+5	2
+C	1
+2	10
+(	6
+)	6
+lo	1
+Q	2
+3	2
+die	225
+1@@	2
+I@@	11
+H@@	348
+U@@	22
+A	1
+K@@	222
+ein	735
+wei	127
+ß@@	217
+ä@@	581
+Fr@@	32
+N@@	49
+ü@@	299
+sch@@	350
+mit	292
+tz@@	170
+z	32
+Mann	278
+einem	481
+auf	323
+einer	280
+L@@	97
+ten	183
+eine	360
+einen	159
+ö@@	61
+Frau	133
+ei	79
+Frau@@	25
+O@@	52
+Man@@	7
+tz	15
+ß	10
+hr	12
+V@@	22
+au	4
+sch	27
+Ü@@	1
+„	3
+“	3
+Ö@@	1
+é	2
+-	2
+:	1