docs: fix return to end token

* Fix formatting

* fix format

* fix clippy error

* bump version

.github/actions/run-rust-python-tests/action.yml

@@ -0,0 +1,69 @@
+name: Run Rust and Python tests
+
+description: Format, clippy, Rust tests (incl. doctests), build Python extension and run pytest
+
+inputs:
+  python-version:
+    description: Python version to use
+    required: false
+    default: "3.11"
+  rust-toolchain:
+    description: Rust toolchain channel
+    required: false
+    default: stable
+
+runs:
+  using: composite
+  steps:
+    - name: Setup Rust toolchain
+      uses: dtolnay/rust-toolchain@stable
+      with:
+        toolchain: ${{ inputs.rust-toolchain }}
+        components: clippy,rustfmt
+
+    - name: Setup Python
+      uses: actions/setup-python@v5
+      with:
+        python-version: ${{ inputs.python-version }}
+
+    - name: Upgrade pip
+      shell: bash
+      run: |
+        python -m pip install --upgrade pip
+
+    - name: Install test deps
+      shell: bash
+      run: |
+        python -m pip install pytest
+
+    - name: Check rustfmt
+      shell: bash
+      run: |
+        cargo fmt --all --check
+
+    - name: Run clippy
+      shell: bash
+      run: |
+        cargo clippy --all-targets --all-features -- -D warnings
+
+    - name: Run Rust tests (unit/integration)
+      shell: bash
+      run: |
+        cargo test --all-targets --all-features
+
+    - name: Run Rust doctests
+      shell: bash
+      run: |
+        cargo test --doc
+
+    - name: Build and install Python package
+      shell: bash
+      run: |
+        pip install .
+
+    - name: Run pytest
+      shell: bash
+      env:
+        PYTHONUTF8: "1"
+      run: |
+        pytest -q

.github/workflows/CI.yml

@@ -19,7 +19,23 @@ permissions:
   contents: read
 
 jobs:
+  tests:
+    name: Tests (fmt, clippy, cargo test, doctest, pytest)
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-14, windows-latest]
+    steps:
+      - uses: actions/checkout@v4
+      - name: Run composite test suite
+        uses: ./.github/actions/run-rust-python-tests
+        with:
+          python-version: "3.11"
+          rust-toolchain: stable
+
   linux:
+    needs: [tests]
     runs-on: ${{ matrix.platform.runner }}
     strategy:
       matrix:
@@ -28,14 +44,12 @@ jobs:
             target: x86_64
           - runner: ubuntu-22.04
             target: x86
-          # - runner: ubuntu-22.04
-          #   target: aarch64
+          - runner: ubuntu-22.04
+            target: aarch64
           - runner: ubuntu-22.04
             target: armv7
-          # - runner: ubuntu-22.04
-          #   target: s390x
-          # - runner: ubuntu-22.04
-          #   target: ppc64le
+          - runner: ubuntu-22.04
+            target: ppc64le
     steps:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v5
@@ -43,6 +57,9 @@ jobs:
           python-version: 3.x
       - name: Build wheels
         uses: PyO3/maturin-action@v1
+        env:
+          # Ensure ring's ARM assembly sees an explicit architecture on aarch64 (glibc)
+          CFLAGS_aarch64_unknown_linux_gnu: -D__ARM_ARCH=8
         with:
           target: ${{ matrix.platform.target }}
           args: --release --out dist --find-interpreter
@@ -55,6 +72,7 @@ jobs:
           path: dist
 
   musllinux:
+    needs: [tests]
     runs-on: ${{ matrix.platform.runner }}
     strategy:
       matrix:
@@ -74,6 +92,9 @@ jobs:
           python-version: 3.x
       - name: Build wheels
         uses: PyO3/maturin-action@v1
+        env:
+          # Ensure ring's ARM assembly sees an explicit architecture on aarch64 (musl)
+          CFLAGS_aarch64_unknown_linux_musl: -D__ARM_ARCH=8
         with:
           target: ${{ matrix.platform.target }}
           args: --release --out dist --find-interpreter
@@ -86,6 +107,7 @@ jobs:
           path: dist
 
   windows:
+    needs: [tests]
     runs-on: ${{ matrix.platform.runner }}
     strategy:
       matrix:
@@ -104,7 +126,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: -F python-binding --release --out dist --find-interpreter
+          args: --release --out dist --find-interpreter
           sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
       - name: Upload wheels
         uses: actions/upload-artifact@v4
@@ -113,6 +135,7 @@ jobs:
           path: dist
 
   macos:
+    needs: [tests]
     runs-on: ${{ matrix.platform.runner }}
     strategy:
       matrix:
@@ -130,7 +153,7 @@ jobs:
         uses: PyO3/maturin-action@v1
         with:
           target: ${{ matrix.platform.target }}
-          args: -F python-binding --release --out dist --find-interpreter
+          args: --release --out dist --find-interpreter
           sccache: ${{ !startsWith(github.ref, 'refs/tags/') }}
       - name: Upload wheels
         uses: actions/upload-artifact@v4
@@ -139,6 +162,7 @@ jobs:
           path: dist
 
   sdist:
+    needs: [tests]
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4

.gitignore

@@ -70,3 +70,6 @@ docs/_build/
 
 # Pyenv
 .python-version
+
+# Avoid ignoring shadcn utils
+!demo/harmony-demo/src/lib

Cargo.lock

@@ -1317,7 +1317,7 @@ checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
 
 [[package]]
 name = "openai-harmony"
-version = "0.0.1"
+version = "0.0.4"
 dependencies = [
  "anyhow",
  "base64",

Cargo.toml

@@ -1,9 +1,10 @@
 [package]
 name = "openai-harmony"
-version = "0.0.1"
+version = "0.0.4"
 edition = "2021"
 license = "Apache-2.0"
 repository = "https://github.com/openai/harmony"
+description = "OpenAI's response format for its open-weight model series gpt-oss"
 
 [lib]
 name = "openai_harmony"
@@ -41,7 +42,10 @@ reqwest = { version = "0.12.5", default-features = false, features = [
 ] }
 futures = "0.3"
 clap = { version = "4", features = ["derive"] }
-pyo3 = { version = "0.25.0", optional = true, features = ["extension-module"] }
+pyo3 = { version = "0.25.0", optional = true, features = [
+    "extension-module",
+    "abi3-py38",
+] }
 wasm-bindgen = { version = "0.2.100", optional = true, features = [
     "serde-serialize",
 ] }

LICENSE

@@ -1,10 +1,11 @@
+
                                  Apache License
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
 
-1.  Definitions.
+   1. Definitions.
 
       "License" shall mean the terms and conditions for use, reproduction,
       and distribution as defined by Sections 1 through 9 of this document.
@@ -63,14 +64,14 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
       on behalf of whom a Contribution has been received by Licensor and
       subsequently incorporated within the Work.
 
-2.  Grant of Copyright License. Subject to the terms and conditions of
+   2. Grant of Copyright License. Subject to the terms and conditions of
       this License, each Contributor hereby grants to You a perpetual,
       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
       copyright license to reproduce, prepare Derivative Works of,
       publicly display, publicly perform, sublicense, and distribute the
       Work and such Derivative Works in Source or Object form.
 
-3.  Grant of Patent License. Subject to the terms and conditions of
+   3. Grant of Patent License. Subject to the terms and conditions of
       this License, each Contributor hereby grants to You a perpetual,
       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
       (except as stated in this section) patent license to make, have made,
@@ -86,7 +87,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
       granted to You under this License for that Work shall terminate
       as of the date such litigation is filed.
 
-4.  Redistribution. You may reproduce and distribute copies of the
+   4. Redistribution. You may reproduce and distribute copies of the
       Work or Derivative Works thereof in any medium, with or without
       modifications, and in Source or Object form, provided that You
       meet the following conditions:
@@ -127,7 +128,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
       reproduction, and distribution of the Work otherwise complies with
       the conditions stated in this License.
 
-5.  Submission of Contributions. Unless You explicitly state otherwise,
+   5. Submission of Contributions. Unless You explicitly state otherwise,
       any Contribution intentionally submitted for inclusion in the Work
       by You to the Licensor shall be under the terms and conditions of
       this License, without any additional terms or conditions.
@@ -135,12 +136,12 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
       the terms of any separate license agreement you may have executed
       with Licensor regarding such Contributions.
 
-6.  Trademarks. This License does not grant permission to use the trade
+   6. Trademarks. This License does not grant permission to use the trade
       names, trademarks, service marks, or product names of the Licensor,
       except as required for reasonable and customary use in describing the
       origin of the Work and reproducing the content of the NOTICE file.
 
-7.  Disclaimer of Warranty. Unless required by applicable law or
+   7. Disclaimer of Warranty. Unless required by applicable law or
       agreed to in writing, Licensor provides the Work (and each
       Contributor provides its Contributions) on an "AS IS" BASIS,
       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
@@ -150,7 +151,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
       appropriateness of using or redistributing the Work and assume any
       risks associated with Your exercise of permissions under this License.
 
-8.  Limitation of Liability. In no event and under no legal theory,
+   8. Limitation of Liability. In no event and under no legal theory,
       whether in tort (including negligence), contract, or otherwise,
       unless required by applicable law (such as deliberate and grossly
       negligent acts) or agreed to in writing, shall any Contributor be
@@ -162,7 +163,7 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
       other commercial damages or losses), even if such Contributor
       has been advised of the possibility of such damages.
 
-9.  Accepting Warranty or Additional Liability. While redistributing
+   9. Accepting Warranty or Additional Liability. While redistributing
       the Work or Derivative Works thereof, You may choose to offer,
       and charge a fee for, acceptance of support, warranty, indemnity,
       or other liability obligations and/or rights consistent with this
@@ -173,9 +174,9 @@ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
       incurred by, or claims asserted against, such Contributor by reason
       of your accepting any such warranty or additional liability.
 
-END OF TERMS AND CONDITIONS
+   END OF TERMS AND CONDITIONS
 
-APPENDIX: How to apply the Apache License to your work.
+   APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
       boilerplate notice, with the fields enclosed by brackets "[]"
@@ -186,16 +187,16 @@ APPENDIX: How to apply the Apache License to your work.
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-Copyright 2025 OpenAI
+   Copyright [yyyy] [name of copyright owner]
 
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
 
        http://www.apache.org/licenses/LICENSE-2.0
 
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -1,12 +0,0 @@
-.PHONY: javascript
-js:
-	# Browser ESM build
-	wasm-pack build --target web --out-dir javascript/dist/web --features wasm-binding --no-default-features
-
-	# Node.js ESM/CJS-compatible build
-	wasm-pack build --target nodejs --out-dir javascript/dist/node --features wasm-binding --no-default-features
-
-
-.PHONY: python-local
-python-local:
-	maturin develop -F python-binding --release

README.md

@@ -8,11 +8,11 @@
 <br>
 </center>
 
-The [gpt-oss models][gpt-oss] were trained on the [harmony response format][harmony-format] for defining conversation structures, generating reasoning output and structuring function calls. If you are not using gpt-oss directly but through an API or a provider like Ollama, you will not have to be concerned about this as your inference solution will handle the formatting. If you are building your own inference solution, this guide will walk you through the prompt format. The format is designed to mimic the OpenAI Responses API, so if you have used that API before, this format should hopefully feel familiar to you. gpt-oss should not be used without using the harmony format as it will not work correctly.
+The [gpt-oss models][gpt-oss] were trained on the [harmony response format][harmony-format] for defining conversation structures, generating reasoning output and structuring function calls. If you are not using gpt-oss directly but through an API or a provider like HuggingFace, Ollama, or vLLM, you will not have to be concerned about this as your inference solution will handle the formatting. If you are building your own inference solution, this guide will walk you through the prompt format. The format is designed to mimic the OpenAI Responses API, so if you have used that API before, this format should hopefully feel familiar to you. gpt-oss should not be used without using the harmony format as it will not work correctly.
 
-The format enables the model to output to multiple different channels for chain of thought, and tool calling premables along with regular responses. It also enables specifying various tool namespaces, and structured outputs along with a clear instruction hierarchy. [Check out the guide][harmony-format] to learn more about the format itself.
+The format enables the model to output to multiple different channels for chain of thought, and tool calling preambles along with regular responses. It also enables specifying various tool namespaces, and structured outputs along with a clear instruction hierarchy. [Check out the guide][harmony-format] to learn more about the format itself.
 
-```
+```text
 <|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.
 Knowledge cutoff: 2024-06
 Current date: 2025-06-28
@@ -20,7 +20,9 @@ Current date: 2025-06-28
 Reasoning: high
 
 # Valid channels: analysis, commentary, final. Channel must be included for every message.
-Calls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Instructions
+Calls to these tools must go to the commentary channel: 'functions'.<|end|>
+
+<|start|>developer<|message|># Instructions
 
 Always respond in riddles
 
@@ -86,7 +88,7 @@ convo = Conversation.from_messages([
     Message.from_role_and_content(
         Role.DEVELOPER,
         DeveloperContent.new().with_instructions("Talk like a pirate!")
-    )
+    ),
     Message.from_role_and_content(Role.USER, "Arrr, how be you?"),
 ])
 tokens = enc.render_conversation_for_completion(convo, Role.ASSISTANT)
@@ -114,12 +116,12 @@ openai-harmony = { git = "https://github.com/openai/harmony" }
 ```rust
 use openai_harmony::chat::{Message, Role, Conversation};
 use openai_harmony::{HarmonyEncodingName, load_harmony_encoding};
+
 fn main() -> anyhow::Result<()> {
     let enc = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss)?;
-    let convo = Conversation::from_messages([
-        Message::from_role_and_content(Role::User, "Hello there!"),
-    ]);
-    let tokens = enc.render_conversation_for_completion(&convo, Role::Assistant)?;
+    let convo =
+        Conversation::from_messages([Message::from_role_and_content(Role::User, "Hello there!")]);
+    let tokens = enc.render_conversation_for_completion(&convo, Role::Assistant, None)?;
     println!("{:?}", tokens);
     Ok(())
 }
@@ -130,7 +132,7 @@ fn main() -> anyhow::Result<()> {
 The majority of the rendering and parsing is built in Rust for performance and exposed to Python
 through thin [`pyo3`](https://pyo3.rs/) bindings.
 
-```
+```text
 ┌──────────────────┐      ┌───────────────────────────┐
 │  Python code     │      │  Rust core (this repo)    │
 │  (dataclasses,   │────► │  • chat / encoding logic  │
@@ -140,7 +142,7 @@ through thin [`pyo3`](https://pyo3.rs/) bindings.
 
 ### Repository layout
 
-```
+```text
 .
 ├── src/                  # Rust crate
 │   ├── chat.rs           # High-level data-structures (Role, Message, …)
@@ -149,7 +151,7 @@ through thin [`pyo3`](https://pyo3.rs/) bindings.
 │   ├── tests.rs          # Canonical Rust test-suite
 │   └── py_module.rs      # PyO3 bindings ⇒ compiled as openai_harmony.*.so
 │
-├── harmony/              # Pure-Python wrapper around the binding
+├── python/openai_harmony/ # Pure-Python wrapper around the binding
 │   └── __init__.py       # Dataclasses + helper API mirroring chat.rs
 │
 ├── tests/                # Python test-suite (1-to-1 port of tests.rs)
@@ -177,10 +179,10 @@ source .venv/bin/activate
 # Install maturin and test dependencies
 pip install maturin pytest mypy ruff  # tailor to your workflow
 # Compile the Rust crate *and* install the Python package in editable mode
-maturin develop -F python-binding --release
+maturin develop --release
 ```
 
-`maturin develop -F python-binding` builds _harmony_ with Cargo, produces a native extension
+`maturin develop` builds _harmony_ with Cargo, produces a native extension
 (`openai_harmony.<abi>.so`) and places it in your virtualenv next to the pure-
 Python wrapper – similar to `pip install -e .` for pure Python projects.
 

demo/harmony-demo/src/lib/utils.ts

@@ -0,0 +1,6 @@
+import { clsx, type ClassValue } from "clsx"
+import { twMerge } from "tailwind-merge"
+
+export function cn(...inputs: ClassValue[]) {
+  return twMerge(clsx(inputs))
+}

docs/format.md

@@ -52,19 +52,14 @@ encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
 
 system_message = (
     SystemContent.new()
-        .with_model_identity(
-            "You are ChatGPT, a large language model trained by OpenAI."
-        )
         .with_reasoning_effort(ReasoningEffort.HIGH)
         .with_conversation_start_date("2025-06-28")
-        .with_knowledge_cutoff("2024-06")
-        .with_required_channels(["analysis", "commentary", "final"])
 )
 
 developer_message = (
     DeveloperContent.new()
         .with_instructions("Always respond in riddles")
-        .with_tools(
+        .with_function_tools(
             [
                 ToolDescription.new(
                     "get_current_weather",
@@ -92,7 +87,7 @@ developer_message = (
 convo = Conversation.from_messages(
     [
         Message.from_role_and_content(Role.SYSTEM, system_message),
-        Message.from_role_and_content(Role.DEVELOPER, system_message),
+        Message.from_role_and_content(Role.DEVELOPER, developer_message),
         Message.from_role_and_content(Role.USER, "What is the weather in Tokyo?"),
         Message.from_role_and_content(
             Role.ASSISTANT,
@@ -101,11 +96,11 @@ convo = Conversation.from_messages(
         Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
         .with_channel("commentary")
         .with_recipient("functions.get_weather")
-        .with_content_type("json"),
+        .with_content_type("<|constrain|> json"),
         Message.from_author_and_content(
             Author.new(Role.TOOL, "functions.lookup_weather"),
             '{ "temperature": 20, "sunny": true }',
-        ),
+        ).with_channel("commentary"),
     ]
 )
 
@@ -229,6 +224,8 @@ Once its done generating it will stop with either a `<|return|>` token indicatin
 
 The `final` channel will contain the answer to your user’s request. Check out the [reasoning section](#reasoning) for more details on the chain-of-thought.
 
+**Implementation note:** `<|return|>` is a decode-time stop token only. When you add the assistant’s generated reply to conversation history for the next turn, replace the trailing `<|return|>` with `<|end|>` so that stored messages are fully formed as `<|start|>{header}<|message|>{content}<|end|>`. Prior messages in prompts should therefore end with `<|end|>`. For supervised targets/training examples, ending with `<|return|>` is appropriate; for persisted history, normalize to `<|end|>`.
+
 ### System message format
 
 The system message is used to provide general information to the system. This is different to what might be considered the “system prompt” in other prompt formats. For that, check out the [developer message format](#developer-message-format).
@@ -305,7 +302,7 @@ And the actual answer is:
 ```
 
 **Important:**  
-The model has not been trained to the same safety standards in the chain-of-thought as it has for final output. You should We recommend not to show the chain-of-thought to your users as they might contain harmful content. [Learn more in the model card](https://openai.com/index/gpt-oss-model-card/).
+The model has not been trained to the same safety standards in the chain-of-thought as it has for final output. We recommend not to show the chain-of-thought to your users as they might contain harmful content. [Learn more in the model card](https://openai.com/index/gpt-oss-model-card/).
 
 #### Handling reasoning output in subsequent sampling
 
@@ -327,7 +324,7 @@ Then the input for the next sampling should be
 
 ```
 <|start|>user<|message|>What is 2 + 2?<|end|>
-<|start|>assistant<|channel|>final<|message|>2 + 2 = 4.<|return|>
+<|start|>assistant<|channel|>final<|message|>2 + 2 = 4.<|end|>
 <|start|>user<|message|>What about 9 / 2?<|end|>
 <|start|>assistant
 ```

pyproject.toml

@@ -12,11 +12,17 @@ classifiers = [
 ]
 dynamic = ["version"]
 dependencies = ["pydantic>=2.11.7"]
+description = "OpenAI's response format for its open-weight model series gpt-oss"
+readme = "README.md"
 
 [project.optional-dependencies]
 demo = ["uvicorn", "fastapi"]
 
 [tool.maturin]
-features = ["pyo3/extension-module"]
+features = ["python-binding", "pyo3/extension-module"]
 module-name = "openai_harmony"
 python-source = "python"
+
+[tool.pytest.ini_options]
+# Only collect tests from the top-level tests directory
+testpaths = ["tests"]

python/openai_harmony/__init__.py

@@ -29,7 +29,7 @@ from typing import (
 )
 
 import re
-from pydantic import BaseModel, Field, root_validator, validator
+from pydantic import BaseModel, Field
 
 # Re-export the low-level Rust bindings under a private name so that we can
 # keep the *public* namespace clean and purely Pythonic.
@@ -425,6 +425,10 @@ class RenderConversationConfig(BaseModel):
     auto_drop_analysis: bool = True
 
 
+class RenderOptions(BaseModel):
+    conversation_has_function_tools: bool = False
+
+
 class HarmonyEncoding:
     """High-level wrapper around the Rust ``PyHarmonyEncoding`` class."""
 
@@ -498,9 +502,20 @@ class HarmonyEncoding:
             config=config_dict,
         )
 
-    def render(self, message: Message) -> List[int]:
+    def render(
+        self, message: Message, render_options: Optional[RenderOptions] = None
+    ) -> List[int]:
         """Render a single message into tokens."""
-        return self._inner.render(message_json=message.to_json())
+        if render_options is None:
+            render_options_dict = {"conversation_has_function_tools": False}
+        else:
+            render_options_dict = {
+                "conversation_has_function_tools": render_options.conversation_has_function_tools
+            }
+
+        return self._inner.render(
+            message_json=message.to_json(), render_options=render_options_dict
+        )
 
     # -- Parsing -------------------------------------------------------
 
@@ -612,6 +627,10 @@ class StreamableParser:
         self._inner.process(token)
         return self
 
+    def process_eos(self) -> "StreamableParser":
+        self._inner.process_eos()
+        return self
+
     @property
     def current_content(self) -> str:
         return self._inner.current_content
@@ -689,6 +708,7 @@ __all__ = [
     "Author",
     "Content",
     "TextContent",
+    "DeveloperContent",
     "ToolDescription",
     "SystemContent",
     "Message",

src/lib.rs

@@ -1,3 +1,5 @@
+#![doc = include_str!("../README.md")]
+
 pub mod chat;
 mod encoding;
 mod registry;

src/py_module.rs

@@ -12,9 +12,6 @@
 //! A thin, typed, user-facing Python wrapper around these low-level bindings is
 //! provided in `harmony/__init__.py`.
 
-// Only compile when the `python-binding` feature is enabled.
-#![cfg(feature = "python-binding")]
-
 use pyo3::prelude::*;
 
 // We need the `Python` type later on.
@@ -34,8 +31,6 @@ use crate::{
     load_harmony_encoding, HarmonyEncodingName,
 };
 
-use serde_json;
-
 /// A thin PyO3 wrapper around the Rust `HarmonyEncoding` struct.
 #[pyclass]
 struct PyHarmonyEncoding {
@@ -183,13 +178,29 @@ impl PyHarmonyEncoding {
     }
 
     /// Render a single message into tokens.
-    fn render(&self, message_json: &str) -> PyResult<Vec<u32>> {
+    fn render(
+        &self,
+        message_json: &str,
+        render_options: Option<Bound<'_, PyDict>>,
+    ) -> PyResult<Vec<u32>> {
         let message: crate::chat::Message = serde_json::from_str(message_json).map_err(|e| {
             PyErr::new::<pyo3::exceptions::PyValueError, _>(format!("invalid message JSON: {e}"))
         })?;
 
+        let rust_options = if let Some(options_dict) = render_options {
+            let conversation_has_function_tools = options_dict
+                .get_item("conversation_has_function_tools")?
+                .and_then(|v| v.extract().ok())
+                .unwrap_or(false);
+            Some(crate::encoding::RenderOptions {
+                conversation_has_function_tools,
+            })
+        } else {
+            None
+        };
+
         self.inner
-            .render(&message)
+            .render(&message, rust_options.as_ref())
             .map_err(|e| PyErr::new::<pyo3::exceptions::PyRuntimeError, _>(e.to_string()))
     }
 
@@ -306,6 +317,13 @@ impl PyStreamableParser {
             .map_err(|e| PyErr::new::<HarmonyError, _>(e.to_string()))
     }
 
+    fn process_eos(&mut self) -> PyResult<()> {
+        self.inner
+            .process_eos()
+            .map(|_| ())
+            .map_err(|e| PyErr::new::<HarmonyError, _>(e.to_string()))
+    }
+
     #[getter]
     fn current_content(&self) -> PyResult<String> {
         self.inner
@@ -386,8 +404,7 @@ fn openai_harmony(_py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> {
             "python" => ToolNamespaceConfig::python(),
             _ => {
                 return Err(PyErr::new::<pyo3::exceptions::PyValueError, _>(format!(
-                    "Unknown tool namespace: {}",
-                    tool
+                    "Unknown tool namespace: {tool}"
                 )));
             }
         };

src/registry.rs

@@ -1,6 +1,6 @@
 use std::{
     collections::{HashMap, HashSet},
-    sync::{atomic::AtomicBool, Arc},
+    sync::Arc,
 };
 
 use crate::{
@@ -76,7 +76,6 @@ pub fn load_harmony_encoding(name: HarmonyEncodingName) -> anyhow::Result<Harmon
                     FormattingToken::EndMessageDoneSampling,
                     FormattingToken::EndMessageAssistantToTool,
                 ]),
-                conversation_has_function_tools: Arc::new(AtomicBool::new(false)),
             })
         }
     }

src/tests.rs

@@ -1,3 +1,5 @@
+use std::path::Path;
+
 use crate::{
     chat::{
         Author, Conversation, DeveloperContent, Message, ReasoningEffort, Role, SystemContent,
@@ -10,12 +12,25 @@ use crate::{
 use pretty_assertions::{assert_eq, Comparison};
 use serde_json::json;
 
-fn parse_tokens(text: &str) -> Vec<Rank> {
-    text.split_whitespace()
+fn parse_tokens(text: impl AsRef<str>) -> Vec<Rank> {
+    text.as_ref()
+        .split_whitespace()
         .map(|s| s.parse().unwrap())
         .collect()
 }
 
+fn load_test_data(path: impl AsRef<Path>) -> String {
+    // on windows, we need to replace \r\n with \n
+    let cargo_manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
+    let src_dir = cargo_manifest_dir.join("src");
+    let path = src_dir.join(path);
+    std::fs::read_to_string(path)
+        .unwrap()
+        .replace("\r\n", "\n")
+        .trim_end()
+        .to_string()
+}
+
 const ENCODINGS: [HarmonyEncodingName; 1] = [HarmonyEncodingName::HarmonyGptOss];
 
 #[test]
@@ -25,7 +40,7 @@ fn test_simple_convo() {
         let expected_tokens = encoding
             .tokenizer
             .encode(
-                include_str!("../test-data/test_simple_convo.txt").trim_end(),
+                load_test_data("../test-data/test_simple_convo.txt").as_str(),
                 &encoding.tokenizer.special_tokens(),
             )
             .0;
@@ -50,45 +65,42 @@ fn test_simple_convo_with_effort() {
     let test_cases = [
         (
             ReasoningEffort::Low,
-            include_str!("../test-data/test_simple_convo_low_effort.txt"),
+            load_test_data("../test-data/test_simple_convo_low_effort.txt"),
             true,
         ),
         (
             ReasoningEffort::Medium,
-            include_str!("../test-data/test_simple_convo_medium_effort.txt"),
+            load_test_data("../test-data/test_simple_convo_medium_effort.txt"),
             true,
         ),
         (
             ReasoningEffort::High,
-            include_str!("../test-data/test_simple_convo_high_effort.txt"),
+            load_test_data("../test-data/test_simple_convo_high_effort.txt"),
             true,
         ),
         (
             ReasoningEffort::Low,
-            include_str!("../test-data/test_simple_convo_low_effort_no_instruction.txt"),
+            load_test_data("../test-data/test_simple_convo_low_effort_no_instruction.txt"),
             false,
         ),
         (
             ReasoningEffort::Medium,
-            include_str!("../test-data/test_simple_convo_medium_effort_no_instruction.txt"),
+            load_test_data("../test-data/test_simple_convo_medium_effort_no_instruction.txt"),
             false,
         ),
         (
             ReasoningEffort::High,
-            include_str!("../test-data/test_simple_convo_high_effort_no_instruction.txt"),
+            load_test_data("../test-data/test_simple_convo_high_effort_no_instruction.txt"),
             false,
         ),
     ];
 
     for encoding_name in ENCODINGS {
         let encoding = load_harmony_encoding(encoding_name).unwrap();
-        for (effort, expected_text, use_instruction) in test_cases {
+        for &(effort, ref expected_text, use_instruction) in &test_cases {
             let expected_tokens = encoding
                 .tokenizer
-                .encode(
-                    expected_text.trim_end(),
-                    &encoding.tokenizer.special_tokens(),
-                )
+                .encode(expected_text.as_str(), &encoding.tokenizer.special_tokens())
                 .0;
             let sys = SystemContent::new()
                 .with_model_identity("You are ChatGPT, a large language model trained by OpenAI.")
@@ -123,8 +135,8 @@ fn test_simple_convo_with_effort() {
 
 #[test]
 fn test_simple_reasoning_response() {
-    let expected_tokens = parse_tokens(include_str!(
-        "../test-data/test_simple_reasoning_response.txt"
+    let expected_tokens = parse_tokens(load_test_data(
+        "../test-data/test_simple_reasoning_response.txt",
     ));
     for encoding_name in ENCODINGS {
         let encoding = load_harmony_encoding(encoding_name).unwrap();
@@ -180,7 +192,7 @@ fn test_reasoning_system_message() {
         let expected = encoding
             .tokenizer
             .encode(
-                include_str!("../test-data/test_reasoning_system_message.txt").trim_end(),
+                load_test_data("../test-data/test_reasoning_system_message.txt").as_str(),
                 &encoding.tokenizer.special_tokens(),
             )
             .0;
@@ -211,8 +223,8 @@ fn test_reasoning_system_message_no_instruction() {
         let expected = encoding
             .tokenizer
             .encode(
-                include_str!("../test-data/test_reasoning_system_message_no_instruction.txt")
-                    .trim_end(),
+                load_test_data("../test-data/test_reasoning_system_message_no_instruction.txt")
+                    .as_str(),
                 &encoding.tokenizer.special_tokens(),
             )
             .0;
@@ -245,8 +257,8 @@ fn test_reasoning_system_message_with_dates() {
         let expected = encoding
             .tokenizer
             .encode(
-                include_str!("../test-data/test_reasoning_system_message_with_dates.txt")
-                    .trim_end(),
+                load_test_data("../test-data/test_reasoning_system_message_with_dates.txt")
+                    .as_str(),
                 &encoding.tokenizer.special_tokens(),
             )
             .0;
@@ -275,8 +287,7 @@ fn test_reasoning_system_message_with_dates() {
 #[test]
 fn test_render_functions_with_parameters() {
     let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
-    let expected_output =
-        include_str!("../test-data/test_render_functions_with_parameters.txt").trim_end();
+    let expected_output = load_test_data("../test-data/test_render_functions_with_parameters.txt");
 
     let sys = SystemContent::new()
         .with_reasoning_effort(ReasoningEffort::High)
@@ -382,7 +393,7 @@ fn test_render_functions_with_parameters() {
 #[test]
 fn test_browser_and_python_tool() {
     let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
-    let expected_output = include_str!("../test-data/test_browser_and_python_tool.txt").trim_end();
+    let expected_output = load_test_data("../test-data/test_browser_and_python_tool.txt");
 
     let convo = Conversation::from_messages([Message::from_role_and_content(
         Role::System,
@@ -403,7 +414,7 @@ fn test_browser_and_python_tool() {
 #[test]
 fn test_dropping_cot_by_default() {
     let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
-    let expected_output = include_str!("../test-data/test_dropping_cot_by_default.txt").trim_end();
+    let expected_output = load_test_data("../test-data/test_dropping_cot_by_default.txt");
 
     let convo = Conversation::from_messages([
         Message::from_role_and_content(Role::User, "What is 2 + 2?"),
@@ -433,8 +444,7 @@ fn test_dropping_cot_by_default() {
 #[test]
 fn test_does_not_drop_if_ongoing_analysis() {
     let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
-    let expected_output =
-        include_str!("../test-data/test_does_not_drop_if_ongoing_analysis.txt").trim_end();
+    let expected_output = load_test_data("../test-data/test_does_not_drop_if_ongoing_analysis.txt");
 
     let convo = Conversation::from_messages([
         Message::from_role_and_content(Role::User, "What is the weather in SF?"),
@@ -470,7 +480,7 @@ fn test_does_not_drop_if_ongoing_analysis() {
 #[test]
 fn test_preserve_cot() {
     let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
-    let expected_output = include_str!("../test-data/test_preserve_cot.txt").trim_end();
+    let expected_output = load_test_data("../test-data/test_preserve_cot.txt");
 
     let convo = Conversation::from_messages([
         Message::from_role_and_content(Role::User, "What is 2 + 2?"),
@@ -501,11 +511,11 @@ fn test_preserve_cot() {
 fn test_reserved_token_decoding() {
     let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
     assert_eq!(
-        encoding.tokenizer.decode_utf8(&[200014]).unwrap(),
+        encoding.tokenizer.decode_utf8([200014]).unwrap(),
         "<|reserved_200014|>"
     );
     assert_eq!(
-        encoding.tokenizer.decode_utf8(&[201088]).unwrap(),
+        encoding.tokenizer.decode_utf8([201088]).unwrap(),
         "<|reserved_201088|>"
     );
 }
@@ -515,7 +525,7 @@ fn test_render_and_render_conversation_roundtrip() {
     let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
     let msg = Message::from_role_and_content(Role::User, "Hello");
     let convo = Conversation::from_messages([msg.clone()]);
-    let tokens_msg = encoding.render(&msg).unwrap();
+    let tokens_msg = encoding.render(&msg, None).unwrap();
     let tokens_convo = encoding.render_conversation(&convo, None).unwrap();
     assert_eq!(tokens_msg, tokens_convo);
     let tokens_completion = encoding
@@ -527,17 +537,17 @@ fn test_render_and_render_conversation_roundtrip() {
 #[test]
 fn test_decode_utf8_invalid_token() {
     let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
-    let result = encoding.tokenizer.decode_utf8(&[99999999]);
+    let result = encoding.tokenizer.decode_utf8([99999999]);
     assert!(result.is_err(), "Expected error for invalid token");
 }
 
 #[test]
 fn test_tool_response_parsing() {
     let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
-    let text_tokens = include_str!("../test-data/test_tool_response_parsing.txt").trim_end();
+    let text_tokens = load_test_data("../test-data/test_tool_response_parsing.txt");
     let tokens = encoding
         .tokenizer
-        .encode(text_tokens, &encoding.tokenizer.special_tokens())
+        .encode(&text_tokens, &encoding.tokenizer.special_tokens())
         .0;
 
     let expected_message = Message::from_author_and_content(
@@ -616,10 +626,10 @@ fn test_invalid_utf8_decoding() {
 #[test]
 fn test_streamable_parser() {
     let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
-    let text = include_str!("../test-data/test_streamable_parser.txt").trim_end();
+    let text = load_test_data("../test-data/test_streamable_parser.txt");
     let tokens = encoding
         .tokenizer
-        .encode(text, &encoding.tokenizer.special_tokens())
+        .encode(&text, &encoding.tokenizer.special_tokens())
         .0;
     let mut parser =
         crate::encoding::StreamableParser::new(encoding.clone(), Some(Role::Assistant)).unwrap();
@@ -663,3 +673,39 @@ fn test_streamable_parser_tool_call_with_constrain_adjacent() {
         parser.messages()[0]
     );
 }
+
+#[test]
+fn test_tool_call_with_constrain_marker_adjacent() {
+    let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
+    let text = "<|start|>assistant to=functions.get_weather<|channel|>commentary<|constrain|>json<|message|>{\"location\": \"Tokyo\"}<|end|>";
+    let tokens = encoding.tokenizer().encode_with_special_tokens(text);
+    let parsed = encoding
+        .parse_messages_from_completion_tokens(tokens, None)
+        .expect("expected to parse");
+    let expected =
+        vec![
+            Message::from_role_and_content(Role::Assistant, "{\"location\": \"Tokyo\"}")
+                .with_channel("commentary")
+                .with_recipient("functions.get_weather")
+                .with_content_type("<|constrain|>json"),
+        ];
+    assert_eq!(parsed, expected);
+}
+
+#[test]
+fn test_tool_call_with_channel_before_recipient_and_constrain_adjacent() {
+    let encoding = load_harmony_encoding(HarmonyEncodingName::HarmonyGptOss).unwrap();
+    let text = "<|start|>assistant<|channel|>commentary to=functions.get_weather<|constrain|>json<|message|>{\"latitude\":48.8566,\"longitude\":2.3522}<|call|>";
+    let tokens = encoding.tokenizer().encode_with_special_tokens(text);
+    let parsed = encoding
+        .parse_messages_from_completion_tokens(tokens, None)
+        .expect("expected to parse");
+    let expected = vec![Message::from_role_and_content(
+        Role::Assistant,
+        "{\"latitude\":48.8566,\"longitude\":2.3522}",
+    )
+    .with_channel("commentary")
+    .with_recipient("functions.get_weather")
+    .with_content_type("<|constrain|>json")];
+    assert_eq!(parsed, expected);
+}

src/wasm_module.rs

@@ -1,5 +1,3 @@
-#![cfg(feature = "wasm-binding")]
-
 use wasm_bindgen::prelude::*;
 
 use crate::{
@@ -9,8 +7,6 @@ use crate::{
 };
 
 use serde::Deserialize;
-use serde_json;
-use serde_wasm_bindgen;
 
 #[wasm_bindgen]
 extern "C" {
@@ -22,6 +18,9 @@ extern "C" {
 
     #[wasm_bindgen(typescript_type = "RenderConversationConfig")]
     pub type JsRenderConversationConfig;
+
+    #[wasm_bindgen(typescript_type = "RenderOptions")]
+    pub type JsRenderOptions;
 }
 
 #[wasm_bindgen(typescript_custom_section)]
@@ -131,12 +130,34 @@ impl JsHarmonyEncoding {
     }
 
     #[wasm_bindgen]
-    pub fn render(&self, message: JsMessage) -> Result<Vec<u32>, JsValue> {
+    pub fn render(
+        &self,
+        message: JsMessage,
+        render_options: JsRenderOptions,
+    ) -> Result<Vec<u32>, JsValue> {
         let message: JsValue = message.into();
         let message: crate::chat::Message = serde_wasm_bindgen::from_value(message)
             .map_err(|e| JsValue::from_str(&format!("invalid message JSON: {e}")))?;
+
+        #[derive(Deserialize)]
+        struct RenderOptions {
+            conversation_has_function_tools: Option<bool>,
+        }
+        let render_options: JsValue = render_options.into();
+        let rust_options = if render_options.is_undefined() || render_options.is_null() {
+            None
+        } else {
+            let cfg: RenderOptions = serde_wasm_bindgen::from_value(render_options)
+                .map_err(|e| JsValue::from_str(&format!("invalid render options: {e}")))?;
+            Some(crate::encoding::RenderOptions {
+                conversation_has_function_tools: cfg
+                    .conversation_has_function_tools
+                    .unwrap_or(false),
+            })
+        };
+
         self.inner
-            .render(&message)
+            .render(&message, rust_options.as_ref())
             .map_err(|e| JsValue::from_str(&e.to_string()))
     }
 
@@ -235,7 +256,7 @@ impl JsStreamableParser {
     pub fn new(encoding: &JsHarmonyEncoding, role: &str) -> Result<JsStreamableParser, JsValue> {
         let parsed_role = Role::try_from(role)
             .map_err(|_| JsValue::from_str(&format!("unknown role: {role}")))?;
-        let inner = StreamableParser::new(encoding.inner.clone(), parsed_role)
+        let inner = StreamableParser::new(encoding.inner.clone(), Some(parsed_role))
             .map_err(|e| JsValue::from_str(&e.to_string()))?;
         Ok(Self { inner })
     }
@@ -323,9 +344,8 @@ pub async fn load_harmony_encoding(
     let parsed: HarmonyEncodingName = name
         .parse::<HarmonyEncodingName>()
         .map_err(|e| JsValue::from_str(&e.to_string()))?;
-    let encoding = inner_load_harmony_encoding(parsed)
-        .await
-        .map_err(|e| JsValue::from_str(&e.to_string()))?;
+    let encoding =
+        inner_load_harmony_encoding(parsed).map_err(|e| JsValue::from_str(&e.to_string()))?;
     Ok(JsHarmonyEncoding { inner: encoding })
 }
 
@@ -336,8 +356,7 @@ pub fn get_tool_namespace_config(tool: &str) -> Result<JsValue, JsValue> {
         "python" => ToolNamespaceConfig::python(),
         _ => {
             return Err(JsValue::from_str(&format!(
-                "Unknown tool namespace: {}",
-                tool
+                "Unknown tool namespace: {tool}"
             )))
         }
     };

test_python.sh

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+set -e
+source .venv/bin/activate
+maturin develop --release 
+pytest "$@"

tests/test_harmony.py

@@ -34,7 +34,6 @@ from openai_harmony import (  # noqa: E402
     StreamableParser,
     SystemContent,
     ToolDescription,
-    ToolNamespaceConfig,
     load_harmony_encoding,
 )
 from pydantic import ValidationError
@@ -60,7 +59,7 @@ def _assert_tokens_eq(encoding, expected: List[int], actual: List[int]):  # type
 
 
 def read_expected_tokens(file_path: Path) -> List[int]:
-    with open(file_path, "r") as f:
+    with open(file_path, "r", encoding="utf-8") as f:
         return [int(x) for x in f.read().split()]
 
 
@@ -79,7 +78,9 @@ def test_simple_convo(encoding_name):
     encoding = load_harmony_encoding(encoding_name)
 
     expected_text = (
-        (ROOT_DIR / "test-data" / "test_simple_convo.txt").read_text().rstrip()
+        (ROOT_DIR / "test-data" / "test_simple_convo.txt")
+        .read_text(encoding="utf-8")
+        .rstrip()
     )
     expected_tokens = encoding.encode(expected_text, allowed_special="all")
 
@@ -144,7 +145,7 @@ def test_simple_convo_with_effort(encoding_name):
     ]
 
     for effort, tokens_path, use_instruction in test_cases:
-        expected_text = tokens_path.read_text().rstrip()
+        expected_text = tokens_path.read_text(encoding="utf-8").rstrip()
         expected_tokens = encoding.encode(expected_text, allowed_special="all")
         sys = (
             SystemContent.new()
@@ -232,6 +233,36 @@ def test_simple_tool_call(encoding_name):
     assert parsed == expected
 
 
+@pytest.mark.parametrize(
+    "encoding_name",
+    [
+        HarmonyEncodingName.HARMONY_GPT_OSS,
+    ],
+)
+def test_tool_call_with_constrain_tokenized_correctly(encoding_name):
+    """
+    Despite passing <|constrain|> as a string in "content_type" it has to be kept as a special token.
+    """
+    encoding = load_harmony_encoding(encoding_name)
+    text = (
+        "<|start|>assistant to=functions.get_weather<|channel|>commentary"
+        ' <|constrain|>json<|message|>{"location": "Tokyo"}<|call|>'
+    )
+    tokens = encoding.encode(text, allowed_special="all")
+    parsed = encoding.parse_messages_from_completion_tokens(tokens, role=None)
+    expected = [
+        Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
+        .with_channel("commentary")
+        .with_recipient("functions.get_weather")
+        .with_content_type("<|constrain|>json"),
+    ]
+    assert parsed == expected
+
+    rendered = encoding.render_conversation(Conversation.from_messages(expected))
+    assert text == encoding.decode_utf8(tokens)
+    assert rendered == tokens
+
+
 @pytest.mark.parametrize(
     "encoding_name",
     [
@@ -245,23 +276,18 @@ def test_tool_call_with_constrain_marker_adjacent(encoding_name):
     correctly and instead handle it as a separate content type.
     """
     encoding = load_harmony_encoding(encoding_name)
-
     text = (
         "<|start|>assistant to=functions.get_weather<|channel|>commentary"
-        '<|constrain|>json<|message|>{"location": "Tokyo"}<|end|>'
+        '<|constrain|>json<|message|>{"location": "Tokyo"}<|call|>'
     )
-
     tokens = encoding.encode(text, allowed_special="all")
-
-    parsed = encoding.parse_messages_from_completion_tokens(tokens, role=Role.ASSISTANT)
-
+    parsed = encoding.parse_messages_from_completion_tokens(tokens, role=None)
     expected = [
         Message.from_role_and_content(Role.ASSISTANT, '{"location": "Tokyo"}')
         .with_channel("commentary")
         .with_recipient("functions.get_weather")
         .with_content_type("<|constrain|>json"),
     ]
-
     assert parsed == expected
 
 
@@ -280,11 +306,8 @@ def test_tool_call_with_channel_before_recipient_and_constrain_adjacent(
         "<|start|>assistant<|channel|>commentary to=functions.get_weather"
         '<|constrain|>json<|message|>{"latitude":48.8566,"longitude":2.3522}<|call|>'
     )
-
     tokens = encoding.encode(text, allowed_special="all")
-
-    parsed = encoding.parse_messages_from_completion_tokens(tokens, role=Role.ASSISTANT)
-
+    parsed = encoding.parse_messages_from_completion_tokens(tokens, role=None)
     expected = [
         Message.from_role_and_content(
             Role.ASSISTANT, '{"latitude":48.8566,"longitude":2.3522}'
@@ -308,7 +331,7 @@ def test_reasoning_system_message(encoding_name):
 
     expected_text = (
         (ROOT_DIR / "test-data" / "test_reasoning_system_message.txt")
-        .read_text()
+        .read_text(encoding="utf-8")
         .rstrip()
     )
     expected = encoding.encode(expected_text, allowed_special="all")
@@ -345,7 +368,7 @@ def test_reasoning_system_message_no_instruction(encoding_name):
 
     expected_text = (
         (ROOT_DIR / "test-data" / "test_reasoning_system_message_no_instruction.txt")
-        .read_text()
+        .read_text(encoding="utf-8")
         .rstrip()
     )
     expected = encoding.encode(expected_text, allowed_special="all")
@@ -385,7 +408,7 @@ def test_reasoning_system_message_with_dates(encoding_name):
 
     expected_text = (
         (ROOT_DIR / "test-data" / "test_reasoning_system_message_with_dates.txt")
-        .read_text()
+        .read_text(encoding="utf-8")
         .rstrip()
     )
     expected = encoding.encode(expected_text, allowed_special="all")
@@ -418,7 +441,7 @@ def test_render_functions_with_parameters():
 
     expected_output = (
         (ROOT_DIR / "test-data" / "test_render_functions_with_parameters.txt")
-        .read_text()
+        .read_text(encoding="utf-8")
         .rstrip()
     )
 
@@ -535,7 +558,9 @@ def test_render_functions_with_parameters():
 def test_no_tools():
     encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
     expected_output = (
-        (ROOT_DIR / "test-data" / "test_no_tools.txt").read_text().rstrip()
+        (ROOT_DIR / "test-data" / "test_no_tools.txt")
+        .read_text(encoding="utf-8")
+        .rstrip()
     )
 
     convo = Conversation.from_messages(
@@ -555,7 +580,9 @@ def test_no_tools():
 def test_browser_tool_only():
     encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
     expected_output = (
-        (ROOT_DIR / "test-data" / "test_browser_tool_only.txt").read_text().rstrip()
+        (ROOT_DIR / "test-data" / "test_browser_tool_only.txt")
+        .read_text(encoding="utf-8")
+        .rstrip()
     )
 
     convo = Conversation.from_messages(
@@ -578,7 +605,7 @@ def test_browser_and_function_tool():
     encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
     expected_output = (
         (ROOT_DIR / "test-data" / "test_browser_and_function_tool.txt")
-        .read_text()
+        .read_text(encoding="utf-8")
         .rstrip()
     )
 
@@ -620,7 +647,7 @@ def test_browser_and_python_tool():
     encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
     expected_output = (
         (ROOT_DIR / "test-data" / "test_browser_and_python_tool.txt")
-        .read_text()
+        .read_text(encoding="utf-8")
         .rstrip()
     )
 
@@ -646,7 +673,7 @@ def test_dropping_cot_by_default():
 
     expected_output = (
         (ROOT_DIR / "test-data" / "test_dropping_cot_by_default.txt")
-        .read_text()
+        .read_text(encoding="utf-8")
         .rstrip()
     )
 
@@ -676,7 +703,7 @@ def test_does_not_drop_if_ongoing_analysis():
 
     expected_output = (
         (ROOT_DIR / "test-data" / "test_does_not_drop_if_ongoing_analysis.txt")
-        .read_text()
+        .read_text(encoding="utf-8")
         .rstrip()
     )
 
@@ -705,13 +732,17 @@ def test_does_not_drop_if_ongoing_analysis():
     )
 
     assert encoding.decode_utf8(tokens) == expected_output
+    # ensure that <|constrain|>json part is tokenized correctly as special tokens
+    assert encoding.encode(expected_output, allowed_special="all") == tokens
 
 
 def test_preserve_cot():
     encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
 
     expected_output = (
-        (ROOT_DIR / "test-data" / "test_preserve_cot.txt").read_text().rstrip()
+        (ROOT_DIR / "test-data" / "test_preserve_cot.txt")
+        .read_text(encoding="utf-8")
+        .rstrip()
     )
 
     convo = Conversation.from_messages(
@@ -747,7 +778,7 @@ def test_keep_analysis_between_final_messages():
 
     expected_output = (
         (ROOT_DIR / "test-data" / "test_keep_analysis_between_finals.txt")
-        .read_text()
+        .read_text(encoding="utf-8")
         .rstrip()
     )
 
@@ -889,7 +920,9 @@ def test_tool_response_parsing():
     encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
 
     text_tokens = (
-        (ROOT_DIR / "test-data" / "test_tool_response_parsing.txt").read_text().rstrip()
+        (ROOT_DIR / "test-data" / "test_tool_response_parsing.txt")
+        .read_text(encoding="utf-8")
+        .rstrip()
     )
 
     expected_message = (
@@ -913,7 +946,9 @@ def test_streamable_parser():
     encoding = load_harmony_encoding(HarmonyEncodingName.HARMONY_GPT_OSS)
 
     text_tokens = (
-        (ROOT_DIR / "test-data" / "test_streamable_parser.txt").read_text().rstrip()
+        (ROOT_DIR / "test-data" / "test_streamable_parser.txt")
+        .read_text(encoding="utf-8")
+        .rstrip()
     )
 
     tokens = encoding.encode(text_tokens, allowed_special="all")