feat: Include token metadata in analysis response

Extend promptTokens and generatedTokens with full metadata:
- bpe_pieces: BPE breakdown of each token
- is_special: Whether token is a special token (eos/bos/pad/unk)
- is_multi_split: Whether token is part of a multi-split identifier
- num_pieces: Number of BPE pieces

This eliminates the need for per-token /token/metadata API calls,
reducing backend requests and enabling instant hover tooltips.

Also removes debug logging from /token/metadata endpoint.

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <[email protected]>

backend/model_service.py

@@ -1860,13 +1860,40 @@ async def analyze_research_attention(request: Dict[str, Any], authenticated: boo
             }
         }
 
+        # Build token metadata for frontend (eliminates per-token API calls)
+        from .tokenizer_utils import TokenizerMetadata
+        token_metadata = TokenizerMetadata(manager.tokenizer)
+
+        special_token_ids = {
+            manager.tokenizer.eos_token_id,
+            manager.tokenizer.bos_token_id,
+            manager.tokenizer.pad_token_id,
+            manager.tokenizer.unk_token_id
+        }
+
+        def build_token_data(token_ids, token_texts, token_type):
+            """Build token data with full metadata for hover tooltips"""
+            multi_split_flags = token_metadata.is_multi_split_identifier(token_ids)
+            result = []
+            for i, (tid, t) in enumerate(zip(token_ids, token_texts)):
+                bpe_pieces = token_metadata.get_subword_pieces(tid)
+                result.append({
+                    "text": t,
+                    "idx": tid,
+                    "bytes": len(t.encode('utf-8')),
+                    "type": token_type,
+                    "bpe_pieces": bpe_pieces,
+                    "is_special": tid in special_token_ids,
+                    "is_multi_split": multi_split_flags[i] if i < len(multi_split_flags) else False,
+                    "num_pieces": len(bpe_pieces),
+                })
+            return result
+
         # Build response
         response = {
             "prompt": prompt,
-            "promptTokens": [{"text": t, "idx": tid, "bytes": len(t.encode('utf-8')), "type": "prompt"}
-                           for tid, t in zip(prompt_token_ids, prompt_tokens)],
-            "generatedTokens": [{"text": t, "idx": tid, "bytes": len(t.encode('utf-8')), "type": "generated"}
-                              for tid, t in zip(generated_token_ids, generated_tokens)],
+            "promptTokens": build_token_data(prompt_token_ids, prompt_tokens, "prompt"),
+            "generatedTokens": build_token_data(generated_token_ids, generated_tokens, "generated"),
             "tokenSections": token_sections,  # Section boundaries for UI coloring
             "tokenAlternatives": token_alternatives_by_step,  # Top-k alternatives for each token
             "layersDataByStep": layer_data_by_token,  # Layer data for ALL generation steps
@@ -2728,21 +2755,7 @@ async def get_token_metadata(
     is_multi_split_array = metadata.is_multi_split_identifier([token_id])
     is_multi_split = is_multi_split_array[0] if is_multi_split_array else False
 
-    # DEBUG LOGGING
-    print(f"\n{'='*60}")
-    print(f"TOKEN METADATA DEBUG - Token ID: {token_id}")
-    print(f"{'='*60}")
-    print(f"Token Text: {repr(token_text)}")
-    print(f"BPE Pieces: {bpe_pieces}")
-    print(f"Num Pieces: {len(bpe_pieces)}")
-    print(f"Byte Length: {byte_length}")
-    print(f"Is Special: {is_special}")
-    print(f"Multi-split Array: {is_multi_split_array}")
-    print(f"Multi-split Boolean: {is_multi_split} (type: {type(is_multi_split).__name__})")
-    print(f"Tokenizer Type: {metadata.tokenizer_type}")
-    print(f"{'='*60}\n")
-
-    result = {
+    return {
         "token_id": token_id,
         "text": token_text,
         "bpe_pieces": bpe_pieces,
@@ -2753,10 +2766,6 @@ async def get_token_metadata(
         "tokenizer_type": metadata.tokenizer_type
     }
 
-    print(f"RESPONSE: {result}\n")
-
-    return result
-
 if __name__ == "__main__":
     import uvicorn
     uvicorn.run(app, host="0.0.0.0", port=8000)