Spaces:
Sleeping
Sleeping
gary-boon
Claude Opus 4.5
commited on
Commit
·
e20ccaf
1
Parent(s):
ed06dcb
Fix: Handle MistralCommonTokenizer pad_token setter
Browse filesMistralCommonTokenizer (used by Devstral) doesn't allow setting
pad_token property. Wrap in try/except to handle this gracefully.
🤖 Generated with [Claude Code](https://claude.com/claude-code)
Co-Authored-By: Claude Opus 4.5 <[email protected]>
- backend/model_service.py +5 -1
backend/model_service.py
CHANGED
|
@@ -227,7 +227,11 @@ class ModelManager:
|
|
| 227 |
|
| 228 |
# Load tokenizer
|
| 229 |
self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
|
| 230 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
| 231 |
|
| 232 |
# For Devstral, also load MistralTokenizer for correct encoding
|
| 233 |
self.mistral_tokenizer = None
|
|
|
|
| 227 |
|
| 228 |
# Load tokenizer
|
| 229 |
self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)
|
| 230 |
+
# Set pad_token if the tokenizer allows it (some like MistralCommonTokenizer don't)
|
| 231 |
+
try:
|
| 232 |
+
self.tokenizer.pad_token = self.tokenizer.eos_token
|
| 233 |
+
except AttributeError:
|
| 234 |
+
logger.info(f"Tokenizer doesn't support setting pad_token (using default)")
|
| 235 |
|
| 236 |
# For Devstral, also load MistralTokenizer for correct encoding
|
| 237 |
self.mistral_tokenizer = None
|