File tree Expand file tree Collapse file tree 2 files changed +23
-20
lines changed Expand file tree Collapse file tree 2 files changed +23
-20
lines changed Original file line number Diff line number Diff line change @@ -8,6 +8,15 @@ oncall("ai_infra_mobile_platform")
8
8
9
9
define_common_targets()
10
10
11
+ runtime.export_file(
12
+ name = "test_tiktoken_tokenizer_model",
13
+ src = "resources/test_tiktoken_tokenizer.model",
14
+ visibility = [
15
+ "//pytorch/tokenizers/...",
16
+ "@EXECUTORCH_CLIENTS",
17
+ ],
18
+ )
19
+
11
20
runtime.export_file(
12
21
name = "test_hf_tokenizer.json",
13
22
src = "resources/test_hf_tokenizer.json",
@@ -17,6 +26,20 @@ runtime.export_file(
17
26
],
18
27
)
19
28
29
+ runtime.python_test(
30
+ name = "test_tiktoken_py",
31
+ srcs = [
32
+ "test_tiktoken.py",
33
+ ],
34
+ resources = {
35
+ ":test_tiktoken_tokenizer_model": "resources/test_tiktoken_tokenizer.model",
36
+ },
37
+ deps = [
38
+ "fbsource//third-party/pypi/blobfile:blobfile",
39
+ "//pytorch/tokenizers/pytorch_tokenizers:tokenizers",
40
+ ],
41
+ )
42
+
20
43
runtime.python_test(
21
44
name = "test_python_bindings",
22
45
srcs = [
Original file line number Diff line number Diff line change @@ -114,26 +114,6 @@ def define_common_targets():
114
114
]),
115
115
)
116
116
117
- runtime .export_file (
118
- name = "test_tiktoken_tokenizer_model" ,
119
- src = "resources/test_tiktoken_tokenizer.model" ,
120
- visibility = ["@EXECUTORCH_CLIENTS" , "//pytorch/tokenizers/..." ],
121
- )
122
-
123
- runtime .python_test (
124
- name = "test_tiktoken_py" ,
125
- srcs = [
126
- "test_tiktoken.py" ,
127
- ],
128
- deps = [
129
- "//pytorch/tokenizers/pytorch_tokenizers:tokenizers" ,
130
- "fbsource//third-party/pypi/blobfile:blobfile" ,
131
- ],
132
- resources = {
133
- ":test_tiktoken_tokenizer_model" : "resources/test_tiktoken_tokenizer.model" ,
134
- },
135
- )
136
-
137
117
runtime .cxx_test (
138
118
name = "test_hf_tokenizer" ,
139
119
srcs = [
You can’t perform that action at this time.
0 commit comments