diff --git a/Cargo.lock b/Cargo.lock index ce97b77..e0b0a71 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1317,7 +1317,7 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" [[package]] name = "openai-harmony" -version = "0.0.3" +version = "0.0.4" dependencies = [ "anyhow", "base64", diff --git a/Cargo.toml b/Cargo.toml index 25d070c..23fa1ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "openai-harmony" -version = "0.0.3" +version = "0.0.4" edition = "2021" license = "Apache-2.0" repository = "https://github.com/openai/harmony" diff --git a/src/encoding.rs b/src/encoding.rs index d57f8ec..6a9305b 100644 --- a/src/encoding.rs +++ b/src/encoding.rs @@ -836,12 +836,13 @@ impl Render for HarmonyEncoding { // finally content type if let Some(content_type) = &message.content_type { // <|constrain|> is a unique case which needs to be tokenized as a special token - if let Some(constrain_marker) = self.mapped_format_token(FormattingToken::ConstrainedFormat) { - if content_type.starts_with(constrain_marker) { + if let Some(constrain_marker) = + self.mapped_format_token(FormattingToken::ConstrainedFormat) + { + if let Some(rest) = content_type.strip_prefix(constrain_marker) { // Render the space, then the constrain marker as a special token, then the rest as text (if any) self.render_text_into(" ", into)?; self.render_formatting_token_into(FormattingToken::ConstrainedFormat, into)?; - let rest = &content_type[constrain_marker.len()..]; if !rest.is_empty() { self.render_text_into(rest, into)?; }