1 parent 13c2165 commit e5b772cCopy full SHA for e5b772c
1 file changed
ext/tiktoken_ruby/src/core_bpe_wrapper.rs
@@ -46,9 +46,7 @@ unsafe extern "C" fn encode_without_gvl(data: *mut c_void) -> *mut c_void {
46
let data = &mut *(data as *mut EncodeData);
47
let core_bpe = &*data.core_bpe;
48
let allowed_special: HashSet<&str> = data.allowed_special.iter().map(|s| s.as_str()).collect();
49
- // tiktoken-rs 0.12.0 made `encode` fallible, returning
50
- // `Result<(Vec<Rank>, usize), EncodeError>`. Keep only the token vector and
51
- // surface any tokenization failure to Ruby.
+ // Keep only the token vector and surface any tokenization failure to Ruby.
52
data.result = core_bpe
53
.encode(&data.text, &allowed_special)
54
.map(|(tokens, _last_piece_token_len)| tokens)
0 commit comments