From 2b377a5f167a74da7a2d132a4f4d7ecfe727d891 Mon Sep 17 00:00:00 2001 From: Jacopo Notarstefano Date: Sat, 18 Jul 2026 06:07:41 -0400 Subject: [PATCH 1/2] Fix autothink and thinkdeeper for transformers>=5 --- optillm/autothink/processor.py | 3 ++- optillm/thinkdeeper.py | 4 ++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/optillm/autothink/processor.py b/optillm/autothink/processor.py index 2fcb4998..52e12778 100644 --- a/optillm/autothink/processor.py +++ b/optillm/autothink/processor.py @@ -243,7 +243,8 @@ def process(self, messages: List[Dict[str, str]]) -> str: thinking_messages, continue_final_message=True, return_tensors="pt" - ).to(self.model.device) + ).input_ids + tokens.to(self.model.device) # Reset and update token history in steering hooks if self.steering_hooks: diff --git a/optillm/thinkdeeper.py b/optillm/thinkdeeper.py index 321e9520..5cc73d37 100644 --- a/optillm/thinkdeeper.py +++ b/optillm/thinkdeeper.py @@ -77,8 +77,8 @@ def reasoning_effort(self, messages) -> str: messages, continue_final_message=True, return_tensors="pt" - ) - tokens = tokens.to(self.model.device) + ).input_ids + tokens.to(self.model.device) kv = DynamicCache() n_thinking_tokens = 0 From 9167e89c286881fbee8677e139d12413a97f9140 Mon Sep 17 00:00:00 2001 From: Asankhaya Sharma Date: Sat, 18 Jul 2026 20:25:50 +0800 Subject: [PATCH 2/2] Keep device transfer when unwrapping BatchEncoding, cover deepconf, bump 0.3.22 Builds on the transformers>=5 fix: apply_chat_template now returns a BatchEncoding, so the tensor must be unwrapped via .input_ids. Two corrections to the original change: 1. Preserve the device transfer. `tokens.to(device)` was called without assigning the result, but Tensor.to() is not in-place -- it returns a new tensor. As written the tokens stayed on CPU, so on CUDA/MPS the subsequent model(input_ids=tokens, ...) would fail with a device mismatch. A CPU-only repro hides this. Chain it instead: .input_ids.to(self.model.device). 2. Apply the same fix to optillm/deepconf/processor.py, which has the identical apply_chat_template(return_tensors="pt") -> model(input_ids=tokens) pattern and was still passing a BatchEncoding. Also updates the README badges (drop the GitHub stars badge, use the pepy.tech downloads badge which reports actual counts) and bumps the version to 0.3.22. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 3 +-- optillm/__init__.py | 2 +- optillm/autothink/processor.py | 3 +-- optillm/deepconf/processor.py | 2 +- optillm/thinkdeeper.py | 3 +-- pyproject.toml | 2 +- 6 files changed, 6 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 845db067..7978468c 100644 --- a/README.md +++ b/README.md @@ -9,9 +9,8 @@

- GitHub stars PyPI version - PyPI downloads + PyPI Downloads License

diff --git a/optillm/__init__.py b/optillm/__init__.py index b18101a6..0aad5ab0 100644 --- a/optillm/__init__.py +++ b/optillm/__init__.py @@ -1,5 +1,5 @@ # Version information -__version__ = "0.3.21" +__version__ = "0.3.22" import os as _os diff --git a/optillm/autothink/processor.py b/optillm/autothink/processor.py index 52e12778..bade0fda 100644 --- a/optillm/autothink/processor.py +++ b/optillm/autothink/processor.py @@ -243,8 +243,7 @@ def process(self, messages: List[Dict[str, str]]) -> str: thinking_messages, continue_final_message=True, return_tensors="pt" - ).input_ids - tokens.to(self.model.device) + ).input_ids.to(self.model.device) # Reset and update token history in steering hooks if self.steering_hooks: diff --git a/optillm/deepconf/processor.py b/optillm/deepconf/processor.py index 17030269..14fc31fb 100644 --- a/optillm/deepconf/processor.py +++ b/optillm/deepconf/processor.py @@ -108,7 +108,7 @@ def generate_single_trace(self, messages: List[Dict[str, str]], messages, return_tensors="pt", add_generation_prompt=True - ).to(self.model.device) + ).input_ids.to(self.model.device) # Initialize generation state kv_cache = DynamicCache() diff --git a/optillm/thinkdeeper.py b/optillm/thinkdeeper.py index 5cc73d37..56a2a8ca 100644 --- a/optillm/thinkdeeper.py +++ b/optillm/thinkdeeper.py @@ -77,8 +77,7 @@ def reasoning_effort(self, messages) -> str: messages, continue_final_message=True, return_tensors="pt" - ).input_ids - tokens.to(self.model.device) + ).input_ids.to(self.model.device) kv = DynamicCache() n_thinking_tokens = 0 diff --git a/pyproject.toml b/pyproject.toml index 5a69a541..7e2159ca 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "optillm" -version = "0.3.21" +version = "0.3.22" description = "An optimizing inference proxy for LLMs." readme = "README.md" license = "Apache-2.0"