Fix Whisper tokenization tests (#68)

After https://huggingface.co/openai/whisper-large-v2/commit/ae4642769ce2ad8fc292556ccea8e901f1530655

The previous version was still cached in my computer :(

Author: pcuenca

Tests/TokenizersTests/FactoryTests.swift

@@ -18,6 +18,6 @@ class FactoryTests: XCTestCase {
     func testWhisper() async throws {
         let tokenizer = try await AutoTokenizer.from(pretrained: "openai/whisper-large-v2")
         let inputIds = tokenizer("Today she took a train to the West")
-        XCTAssertEqual(inputIds, [50258, 50363, 27676, 750, 220, 83, 1212, 257, 220, 83, 7146, 220, 1353, 220, 3322, 4055, 50257])
+        XCTAssertEqual(inputIds, [50258, 50363, 27676, 750, 1890, 257, 3847, 281, 264, 4055, 50257])
     }
 }

Tests/TokenizersTests/Resources/whisper_large_v2_encoded.json

@@ -1 +1 @@
-{"text": "Fatouville-Grestain est une commune du Nord-Ouest du d\u00e9partement de l'Eure situ\u00e9e au \nbord de l'estuaire de la Seine et \u00e0 proximit\u00e9 du d\u00e9partement du Calvados. Selon l'atlas des paysages \nde Haute-Normandie, elle appartient \u00e0 la r\u00e9gion naturelle du Lieuvin. Toutefois, l'Agreste, le service \nde la statistique et de la prospective du minist\u00e8re de l'Agriculture, de l'Agroalimentaire et de la For\u00eat, \nla classe au sein du pays d'Auge (en tant que r\u00e9gion agricole).La commune est \u00e0 moins de dix kilom\u00e8tres \u00e0 \nl'est de Honfleur, \u00e0 autant de Beuzeville et \u00e0 environ dix-sept kilom\u00e8tres de Pont-Audemer.", "bpe_tokens": ["F", "at", "ou", "ville", "-", "G", "rest", "ain", "\u0120est", "\u0120une", "\u0120commun", "e", "\u0120du", "\u0120Nord", "-", "O", "uest", "\u0120du", "\u0120d\u00c3\u00a9part", "ement", "\u0120de", "\u0120l", "'", "E", "ure", "\u0120situ", "\u00c3\u00a9e", "\u0120au", "\u0120", "\u010a", "b", "ord", "\u0120de", "\u0120l", "'", "est", "ua", "ire", "\u0120de", "\u0120la", "\u0120Se", "ine", "\u0120et", "\u0120\u00c3\u0142", "\u0120proxim", "it\u00c3\u00a9", "\u0120du", "\u0120d\u00c3\u00a9part", "ement", "\u0120du", "\u0120Cal", "v", "ados", ".", "\u0120Sel", "on", "\u0120l", "'", "at", "las", "\u0120des", "\u0120pays", "ages", "\u0120", "\u010a", "de", "\u0120Ha", "ute", "-", "N", "orm", "and", "ie", ",", "\u0120elle", "\u0120app", "art", "ient", "\u0120\u00c3\u0142", "\u0120la", "\u0120r\u00c3\u00a9gion", "\u0120nature", "lle", "\u0120du", "\u0120Lie", "u", "vin", ".", "\u0120Tout", "ef", "ois", ",", "\u0120l", "'", "A", "gre", "ste", ",", "\u0120le", "\u0120service", "\u0120", "\u010a", "de", "\u0120la", "\u0120statist", "ique", "\u0120et", "\u0120de", "\u0120la", "\u0120prospective", "\u0120du", "\u0120minist", "\u00c3\u00a8re", "\u0120de", "\u0120l", "'", "A", "gr", "ic", "ulture", ",", "\u0120de", "\u0120l", "'", "A", "gro", "al", "iment", "aire", "\u0120et", "\u0120de", "\u0120la", "\u0120For", "\u00c3\u00aat", ",", "\u0120", "\u010a", "la", "\u0120classe", "\u0120au", "\u0120sein", "\u0120du", "\u0120pays", "\u0120d", "'", "A", "uge", "\u0120(", "en", "\u0120", "tant", "\u0120que", "\u0120r\u00c3\u00a9gion", "\u0120agric", "ole", ").", "La", "\u0120commun", "e", "\u0120est", "\u0120\u00c3\u0142", "\u0120moins", "\u0120de", "\u0120d", "ix", "\u0120kil", "om", "\u00c3\u00a8tres", "\u0120\u00c3\u0142", "\u0120", "\u010a", "l", "'", "est", "\u0120de", "\u0120Hon", "fle", "ur", ",", "\u0120\u00c3\u0142", "\u0120autant", "\u0120de", "\u0120Be", "u", "ze", "ville", "\u0120et", "\u0120\u00c3\u0142", "\u0120environ", "\u0120d", "ix", "-", "se", "pt", "\u0120kil", "om", "\u00c3\u00a8tres", "\u0120de", "\u0120Pont", "-", "Aud", "emer", "."], "token_ids": [50258, 50363, 37, 267, 263, 8386, 12, 38, 4149, 491, 871, 2251, 1199, 68, 1581, 16229, 12, 46, 11493, 1581, 37745, 1712, 368, 287, 6, 36, 540, 2054, 3856, 1609, 220, 198, 65, 765, 368, 287, 6, 377, 4398, 621, 368, 635, 1100, 533, 1030, 1531, 21932, 5066, 1581, 37745, 1712, 1581, 3511, 85, 4181, 13, 10736, 266, 287, 6, 267, 7743, 730, 10604, 1660, 220, 198, 1479, 4064, 1169, 12, 45, 687, 474, 414, 11, 8404, 724, 446, 1196, 1531, 635, 42669, 3687, 2447, 1581, 11197, 84, 4796, 13, 20453, 5666, 7376, 11, 287, 6, 32, 33248, 2941, 11, 476, 2643, 220, 198, 1479, 635, 16012, 1925, 1030, 368, 635, 39377, 1581, 16182, 4212, 368, 287, 6, 32, 861, 299, 8389, 11, 368, 287, 6, 32, 17427, 304, 2328, 9020, 1030, 368, 635, 1171, 15519, 11, 220, 198, 875, 32400, 1609, 6195, 1581, 10604, 274, 6, 32, 7181, 522, 268, 220, 35717, 631, 42669, 9682, 4812, 3050, 5478, 1199, 68, 871, 1531, 13099, 368, 274, 970, 5128, 298, 36506, 1531, 220, 198, 75, 6, 377, 368, 6625, 38921, 374, 11, 1531, 34081, 368, 879, 84, 1381, 8386, 1030, 1531, 2571, 274, 970, 12, 405, 662, 5128, 298, 36506, 368, 41127, 12, 15591, 29660, 13, 50257], "decoded_text": "<|startoftranscript|><|notimestamps|>Fatouville-Grestain est une commune du Nord-Ouest du d\u00e9partement de l'Eure situ\u00e9e au \nbord de l'estuaire de la Seine et \u00e0 proximit\u00e9 du d\u00e9partement du Calvados. Selon l'atlas des paysages \nde Haute-Normandie, elle appartient \u00e0 la r\u00e9gion naturelle du Lieuvin. Toutefois, l'Agreste, le service \nde la statistique et de la prospective du minist\u00e8re de l'Agriculture, de l'Agroalimentaire et de la For\u00eat, \nla classe au sein du pays d'Auge (en tant que r\u00e9gion agricole).La commune est \u00e0 moins de dix kilom\u00e8tres \u00e0 \nl'est de Honfleur, \u00e0 autant de Beuzeville et \u00e0 environ dix-sept kilom\u00e8tres de Pont-Audemer.<|endoftext|>"}
+{"text": "Fatouville-Grestain est une commune du Nord-Ouest du d\u00e9partement de l'Eure situ\u00e9e au \nbord de l'estuaire de la Seine et \u00e0 proximit\u00e9 du d\u00e9partement du Calvados. Selon l'atlas des paysages \nde Haute-Normandie, elle appartient \u00e0 la r\u00e9gion naturelle du Lieuvin. Toutefois, l'Agreste, le service \nde la statistique et de la prospective du minist\u00e8re de l'Agriculture, de l'Agroalimentaire et de la For\u00eat, \nla classe au sein du pays d'Auge (en tant que r\u00e9gion agricole).La commune est \u00e0 moins de dix kilom\u00e8tres \u00e0 \nl'est de Honfleur, \u00e0 autant de Beuzeville et \u00e0 environ dix-sept kilom\u00e8tres de Pont-Audemer.", "bpe_tokens": ["F", "at", "ou", "ville", "-", "G", "rest", "ain", "\u0120est", "\u0120une", "\u0120commun", "e", "\u0120du", "\u0120Nord", "-", "O", "uest", "\u0120du", "\u0120d\u00c3\u00a9part", "ement", "\u0120de", "\u0120l", "'", "E", "ure", "\u0120situ", "\u00c3\u00a9e", "\u0120au", "\u0120", "\u010a", "b", "ord", "\u0120de", "\u0120l", "'", "est", "ua", "ire", "\u0120de", "\u0120la", "\u0120Se", "ine", "\u0120et", "\u0120\u00c3\u0142", "\u0120proxim", "it\u00c3\u00a9", "\u0120du", "\u0120d\u00c3\u00a9part", "ement", "\u0120du", "\u0120Cal", "v", "ados", ".", "\u0120Sel", "on", "\u0120l", "'", "at", "las", "\u0120des", "\u0120pays", "ages", "\u0120", "\u010a", "de", "\u0120Ha", "ute", "-", "N", "orm", "and", "ie", ",", "\u0120elle", "\u0120app", "art", "ient", "\u0120\u00c3\u0142", "\u0120la", "\u0120r\u00c3\u00a9gion", "\u0120nature", "lle", "\u0120du", "\u0120Lie", "u", "vin", ".", "\u0120Tout", "ef", "ois", ",", "\u0120l", "'", "A", "gre", "ste", ",", "\u0120le", "\u0120service", "\u0120", "\u010a", "de", "\u0120la", "\u0120statist", "ique", "\u0120et", "\u0120de", "\u0120la", "\u0120prospective", "\u0120du", "\u0120minist", "\u00c3\u00a8re", "\u0120de", "\u0120l", "'", "A", "gr", "ic", "ulture", ",", "\u0120de", "\u0120l", "'", "A", "gro", "al", "iment", "aire", "\u0120et", "\u0120de", "\u0120la", "\u0120For", "\u00c3\u00aat", ",", "\u0120", "\u010a", "la", "\u0120classe", "\u0120au", "\u0120sein", "\u0120du", "\u0120pays", "\u0120d", "'", "A", "uge", "\u0120(", "en", "\u0120tant", "\u0120que", "\u0120r\u00c3\u00a9gion", "\u0120agric", "ole", ").", "La", "\u0120commun", "e", "\u0120est", "\u0120\u00c3\u0142", "\u0120moins", "\u0120de", "\u0120d", "ix", "\u0120kil", "om", "\u00c3\u00a8tres", "\u0120\u00c3\u0142", "\u0120", "\u010a", "l", "'", "est", "\u0120de", "\u0120Hon", "fle", "ur", ",", "\u0120\u00c3\u0142", "\u0120autant", "\u0120de", "\u0120Be", "u", "ze", "ville", "\u0120et", "\u0120\u00c3\u0142", "\u0120environ", "\u0120d", "ix", "-", "se", "pt", "\u0120kil", "om", "\u00c3\u00a8tres", "\u0120de", "\u0120Pont", "-", "Aud", "emer", "."], "token_ids": [50258, 50363, 37, 267, 263, 8386, 12, 38, 4149, 491, 871, 2251, 1199, 68, 1581, 16229, 12, 46, 11493, 1581, 37745, 1712, 368, 287, 6, 36, 540, 2054, 3856, 1609, 220, 198, 65, 765, 368, 287, 6, 377, 4398, 621, 368, 635, 1100, 533, 1030, 1531, 21932, 5066, 1581, 37745, 1712, 1581, 3511, 85, 4181, 13, 10736, 266, 287, 6, 267, 7743, 730, 10604, 1660, 220, 198, 1479, 4064, 1169, 12, 45, 687, 474, 414, 11, 8404, 724, 446, 1196, 1531, 635, 42669, 3687, 2447, 1581, 11197, 84, 4796, 13, 20453, 5666, 7376, 11, 287, 6, 32, 33248, 2941, 11, 476, 2643, 220, 198, 1479, 635, 16012, 1925, 1030, 368, 635, 39377, 1581, 16182, 4212, 368, 287, 6, 32, 861, 299, 8389, 11, 368, 287, 6, 32, 17427, 304, 2328, 9020, 1030, 368, 635, 1171, 15519, 11, 220, 198, 875, 32400, 1609, 6195, 1581, 10604, 274, 6, 32, 7181, 522, 268, 12095, 631, 42669, 9682, 4812, 3050, 5478, 1199, 68, 871, 1531, 13099, 368, 274, 970, 5128, 298, 36506, 1531, 220, 198, 75, 6, 377, 368, 6625, 38921, 374, 11, 1531, 34081, 368, 879, 84, 1381, 8386, 1030, 1531, 2571, 274, 970, 12, 405, 662, 5128, 298, 36506, 368, 41127, 12, 15591, 29660, 13, 50257], "decoded_text": "<|startoftranscript|><|notimestamps|>Fatouville-Grestain est une commune du Nord-Ouest du d\u00e9partement de l'Eure situ\u00e9e au \nbord de l'estuaire de la Seine et \u00e0 proximit\u00e9 du d\u00e9partement du Calvados. Selon l'atlas des paysages \nde Haute-Normandie, elle appartient \u00e0 la r\u00e9gion naturelle du Lieuvin. Toutefois, l'Agreste, le service \nde la statistique et de la prospective du minist\u00e8re de l'Agriculture, de l'Agroalimentaire et de la For\u00eat, \nla classe au sein du pays d'Auge (en tant que r\u00e9gion agricole).La commune est \u00e0 moins de dix kilom\u00e8tres \u00e0 \nl'est de Honfleur, \u00e0 autant de Beuzeville et \u00e0 environ dix-sept kilom\u00e8tres de Pont-Audemer.<|endoftext|>"}