From c505a03e9c9a388a511b6125756097eee523742a Mon Sep 17 00:00:00 2001 From: Soumyadip Sarkar Date: Wed, 6 Aug 2025 04:50:17 +0530 Subject: [PATCH] fix: `meta_sep` token and add to registry --- src/registry.rs | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/registry.rs b/src/registry.rs index 6d8a98f..4fe585d 100644 --- a/src/registry.rs +++ b/src/registry.rs @@ -66,6 +66,8 @@ pub fn load_harmony_encoding(name: HarmonyEncodingName) -> anyhow::Result"), (FormattingToken::BeginUntrusted, "<|untrusted|>"), (FormattingToken::EndUntrusted, "<|end_untrusted|>"), + (FormattingToken::MetaSep, "<|meta_sep|>"), + (FormattingToken::MetaEnd, "<|meta_end|>"), ]), stop_formatting_tokens: HashSet::from([ FormattingToken::EndMessageDoneSampling, @@ -107,6 +109,8 @@ pub async fn load_harmony_encoding(name: HarmonyEncodingName) -> anyhow::Result< (FormattingToken::EndMessageAssistantToTool, "<|call|>"), (FormattingToken::BeginUntrusted, "<|untrusted|>"), (FormattingToken::EndUntrusted, "<|end_untrusted|>"), + (FormattingToken::MetaSep, "<|meta_sep|>"), + (FormattingToken::MetaEnd, "<|meta_end|>"), ]), stop_formatting_tokens: HashSet::from([ FormattingToken::EndMessageDoneSampling,