Use pretrained model for huggingface tokenizer and add equivalence test

Author: hendrikvanantwerpen

crates/bpe/benchmarks/Cargo.toml

@@ -1,17 +1,26 @@
 [package]
-name = "bpe-benches"
+name = "bpe-benchmarks"
 edition = "2021"
 
+[lib]
+path = "lib.rs"
+
 [[bench]]
 name = "performance"
 path = "performance.rs"
 harness = false
 test = false
 
-[dev-dependencies]
+[[test]]
+name = "equivalence"
+path = "equivalence.rs"
+harness = true
+test = true
+
+[dependencies]
 bpe = { path = "../../bpe", features = ["rand", "tiktoken-rs"] }
 bpe-openai = { path = "../../bpe-openai" }
 criterion = "0.5"
 rand = "0.8"
 tiktoken-rs = "0.5"
-tokenizers = "0.20"
+tokenizers = { version = "0.20", features = ["http"] }