Merge pull request #838 from Kaggle/fix-allennlp

Fix allennlp test following major version upgrade

Author: rosbo

Dockerfile

@@ -373,8 +373,7 @@ RUN pip install bcolz && \
     pip install fastai && \
     pip install torchtext && \
     pip install allennlp && \
-    # b/149359379 remove once allennlp 1.0 is released which won't cause a spacy downgrade.
-    pip install spacy==2.2.3 && python -m spacy download en && python -m spacy download en_core_web_lg && \
+    python -m spacy download en && python -m spacy download en_core_web_lg && \
     apt-get install -y ffmpeg && \
     /tmp/clean-layer.sh
 

tests/test_allennlp.py

@@ -1,15 +1,15 @@
 import unittest
 
-from allennlp.data.tokenizers import WordTokenizer
+from allennlp.data.tokenizers import SpacyTokenizer
 
 
 class TestAllenNlp(unittest.TestCase):
     # reference
     # https://github.com/allenai/allennlp/blob/master/allennlp/tests/data/tokenizers/word_tokenizer_test.py
     def test_passes_through_correctly(self):
-        tokenizer = WordTokenizer(start_tokens=['@@', '%%'], end_tokens=['^^'])
+        tokenizer = SpacyTokenizer()
         sentence = "this (sentence) has 'crazy' \"punctuation\"."
         tokens = [t.text for t in tokenizer.tokenize(sentence)]
-        expected_tokens = ["@@", "%%", "this", "(", "sentence", ")", "has", "'", "crazy", "'", "\"",
-                           "punctuation", "\"", ".", "^^"]
-        assert tokens == expected_tokens
+        expected_tokens = ["this", "(", "sentence", ")", "has", "'", "crazy", "'", "\"",
+                           "punctuation", "\"", "."]
+        self.assertSequenceEqual(tokens, expected_tokens)