Skip to content

Commit e5b772c

Browse files
authored
Refine comment in encode_without_gvl function
Updated comment to clarify the purpose of the code.
1 parent 13c2165 commit e5b772c

1 file changed

Lines changed: 1 addition & 3 deletions

File tree

ext/tiktoken_ruby/src/core_bpe_wrapper.rs

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,9 +46,7 @@ unsafe extern "C" fn encode_without_gvl(data: *mut c_void) -> *mut c_void {
4646
let data = &mut *(data as *mut EncodeData);
4747
let core_bpe = &*data.core_bpe;
4848
let allowed_special: HashSet<&str> = data.allowed_special.iter().map(|s| s.as_str()).collect();
49-
// tiktoken-rs 0.12.0 made `encode` fallible, returning
50-
// `Result<(Vec<Rank>, usize), EncodeError>`. Keep only the token vector and
51-
// surface any tokenization failure to Ruby.
49+
// Keep only the token vector and surface any tokenization failure to Ruby.
5250
data.result = core_bpe
5351
.encode(&data.text, &allowed_special)
5452
.map(|(tokens, _last_piece_token_len)| tokens)

0 commit comments

Comments
 (0)