Skip to content

Commit edf25c4

Browse files
larryliu0820larryliu0820
authored
Bump tokenizers pin
Differential Revision: D78123740 Pull Request resolved: #12372
1 parent 3afd18d commit edf25c4

File tree

4 files changed

+11
-1
lines changed

4 files changed

+11
-1
lines changed

‎examples/demo-apps/apple_ios/LLaMA/LLaMA.xcodeproj/project.pbxproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
306A71512DC1DC3D00936B1F /* pre_tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 306A71472DC1DC3D00936B1F /* pre_tokenizer.cpp */; };
6565
306A71522DC1DC3D00936B1F /* token_decoder.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 306A714B2DC1DC3D00936B1F /* token_decoder.cpp */; };
6666
3072D5232DC3EA280083FC83 /* Constants.swift in Sources */ = {isa = PBXBuildFile; fileRef = 3072D5222DC3EA280083FC83 /* Constants.swift */; };
67+
F24909E82E207004001E5B69 /* normalizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F24909E72E207004001E5B69 /* normalizer.cpp */; };
6768
F292B0752D88B0C200BE6839 /* tiktoken.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B06F2D88B0C200BE6839 /* tiktoken.cpp */; };
6869
F292B0762D88B0C200BE6839 /* llama2c_tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B06C2D88B0C200BE6839 /* llama2c_tokenizer.cpp */; };
6970
F292B0772D88B0C200BE6839 /* bpe_tokenizer_base.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B06A2D88B0C200BE6839 /* bpe_tokenizer_base.cpp */; };
@@ -152,6 +153,7 @@
152153
306A714A2DC1DC3D00936B1F /* std_regex.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = std_regex.cpp; path = src/std_regex.cpp; sourceTree = "<group>"; };
153154
306A714B2DC1DC3D00936B1F /* token_decoder.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = token_decoder.cpp; path = src/token_decoder.cpp; sourceTree = "<group>"; };
154155
3072D5222DC3EA280083FC83 /* Constants.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = Constants.swift; sourceTree = "<group>"; };
156+
F24909E72E207004001E5B69 /* normalizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = normalizer.cpp; path = src/normalizer.cpp; sourceTree = "<group>"; };
155157
F292B06A2D88B0C200BE6839 /* bpe_tokenizer_base.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = bpe_tokenizer_base.cpp; path = src/bpe_tokenizer_base.cpp; sourceTree = "<group>"; };
156158
F292B06C2D88B0C200BE6839 /* llama2c_tokenizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = llama2c_tokenizer.cpp; path = src/llama2c_tokenizer.cpp; sourceTree = "<group>"; };
157159
F292B06F2D88B0C200BE6839 /* tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = tiktoken.cpp; path = src/tiktoken.cpp; sourceTree = "<group>"; };
@@ -309,6 +311,7 @@
309311
03729F0E2BB203D700152F2E /* tokenizers */ = {
310312
isa = PBXGroup;
311313
children = (
314+
F24909E72E207004001E5B69 /* normalizer.cpp */,
312315
F292B06A2D88B0C200BE6839 /* bpe_tokenizer_base.cpp */,
313316
306A71452DC1DC3D00936B1F /* hf_tokenizer.cpp */,
314317
F292B1002D88B20C00BE6839 /* llama_tiktoken.cpp */,
@@ -598,6 +601,7 @@
598601
files = (
599602
03D151B82E0E0908007A38BE /* LLaVARunner.mm in Sources */,
600603
03729EE12BB1F93800152F2E /* LLaMARunner.mm in Sources */,
604+
F24909E82E207004001E5B69 /* normalizer.cpp in Sources */,
601605
0372C3152C89418E00CD942A /* llava_runner.cpp in Sources */,
602606
03D151CA2E0E98C4007A38BE /* sentencepiece.cpp in Sources */,
603607
03D151D92E0E9E43007A38BE /* ExecuTorchTextLLMRunner.mm in Sources */,

‎extension/benchmark/apple/Benchmark/Benchmark.xcodeproj/project.pbxproj

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
30AA4B662DC0766800B1BE50 /* re2_regex.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 30AA4B5C2DC0766800B1BE50 /* re2_regex.cpp */; };
3636
3C6ABD332DFA27DE0015DE55 /* regex_lookahead.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 3C6ABD322DFA27DE0015DE55 /* regex_lookahead.cpp */; };
3737
F22E9E1A2DF2CBB900EC5425 /* text_llm_runner.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F22E9E192DF2CBB900EC5425 /* text_llm_runner.cpp */; };
38+
F24909E22E206FBA001E5B69 /* normalizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F24909E12E206FBA001E5B69 /* normalizer.cpp */; };
3839
F292B01D2D88AF3500BE6839 /* bpe_tokenizer_base.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B0162D88AF3500BE6839 /* bpe_tokenizer_base.cpp */; };
3940
F292B0202D88AF3500BE6839 /* llama2c_tokenizer.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B0172D88AF3500BE6839 /* llama2c_tokenizer.cpp */; };
4041
F292B0212D88AF3500BE6839 /* tiktoken.cpp in Sources */ = {isa = PBXBuildFile; fileRef = F292B01A2D88AF3500BE6839 /* tiktoken.cpp */; };
@@ -100,6 +101,7 @@
100101
3C6ABD322DFA27DE0015DE55 /* regex_lookahead.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = regex_lookahead.cpp; path = src/regex_lookahead.cpp; sourceTree = "<group>"; };
101102
F22E9E182DF2CBB900EC5425 /* text_llm_runner.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = text_llm_runner.h; sourceTree = "<group>"; };
102103
F22E9E192DF2CBB900EC5425 /* text_llm_runner.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = text_llm_runner.cpp; sourceTree = "<group>"; };
104+
F24909E12E206FBA001E5B69 /* normalizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = normalizer.cpp; path = src/normalizer.cpp; sourceTree = "<group>"; };
103105
F292B0162D88AF3500BE6839 /* bpe_tokenizer_base.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = bpe_tokenizer_base.cpp; path = src/bpe_tokenizer_base.cpp; sourceTree = "<group>"; };
104106
F292B0172D88AF3500BE6839 /* llama2c_tokenizer.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = llama2c_tokenizer.cpp; path = src/llama2c_tokenizer.cpp; sourceTree = "<group>"; };
105107
F292B01A2D88AF3500BE6839 /* tiktoken.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; name = tiktoken.cpp; path = src/tiktoken.cpp; sourceTree = "<group>"; };
@@ -185,6 +187,7 @@
185187
032A74022CAFBB7800932D36 /* tokenizers */ = {
186188
isa = PBXGroup;
187189
children = (
190+
F24909E12E206FBA001E5B69 /* normalizer.cpp */,
188191
F2E1B5162E03AC19002C9718 /* sentencepiece.cpp */,
189192
3C6ABD322DFA27DE0015DE55 /* regex_lookahead.cpp */,
190193
30AA4B592DC0766800B1BE50 /* hf_tokenizer.cpp */,
@@ -430,6 +433,7 @@
430433
F292B0202D88AF3500BE6839 /* llama2c_tokenizer.cpp in Sources */,
431434
F292B0212D88AF3500BE6839 /* tiktoken.cpp in Sources */,
432435
F2E1B5172E03AC19002C9718 /* sentencepiece.cpp in Sources */,
436+
F24909E22E206FBA001E5B69 /* normalizer.cpp in Sources */,
433437
03E7E6792CBDCAE900205E71 /* CoreMLTests.mm in Sources */,
434438
032A74232CAFC1B300932D36 /* runner.cpp in Sources */,
435439
03B2D37A2C8A515C0046936E /* GenericTests.mm in Sources */,

‎extension/llm/tokenizers

‎third-party/CMakeLists.txt

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@
44
# This source code is licensed under the BSD-style license found in the
55
# LICENSE file in the root directory of this source tree.
66

7+
set(CMAKE_POLICY_VERSION_MINIMUM 3.5)
78
add_subdirectory(json)
89
add_subdirectory(gflags)
910

@@ -86,6 +87,7 @@ ExternalProject_Add(
8687
-DFLATCC_REFLECTION=OFF
8788
-DFLATCC_DEBUG_CLANG_SANITIZE=OFF
8889
-DFLATCC_INSTALL=ON
90+
-DCMAKE_POLICY_VERSION_MINIMUM=3.5
8991
-DCMAKE_INSTALL_PREFIX:PATH=<INSTALL_DIR>
9092
-DCMAKE_POSITION_INDEPENDENT_CODE=ON
9193
-DCMAKE_TOOLCHAIN_FILE=

0 commit comments

Comments
 (0)