window fix again

This commit is contained in:
Scott Lessans 2025-08-05 12:11:49 -07:00
parent eaa1be0286
commit 69001b7064

View file

@ -25,7 +25,9 @@ fn test_simple_convo() {
let expected_tokens = encoding
.tokenizer
.encode(
include_str!("../test-data/test_simple_convo.txt").trim_end(),
include_str!("../test-data/test_simple_convo.txt")
.replace("\r\n", "\n")
.trim_end(),
&encoding.tokenizer.special_tokens(),
)
.0;