lvyufeng
/

DeepSeek-OCR

Image-Text-to-Text

feature-extraction

vision-language

Model card Files Files and versions

lvyufeng commited on 18 days ago

Commit

87108be

·

verified ·

1 Parent(s): 785205e

Update modeling_deepseekocr.py

Files changed (1) hide show

modeling_deepseekocr.py +3 -3

modeling_deepseekocr.py CHANGED Viewed

@@ -1067,9 +1067,9 @@ class DeepseekOCRForCausalLM(DeepseekV2ForCausalLM):
             if isinstance(layer.mlp, DeepseekV2MoE):
                 moe_layer = layer.mlp
                 # combine experts
-                moe_layer.w1 = nn.Parameter(torch.stack([moe_layer.experts[i].gate_proj.weight.T for i in range(moe_layer.config.n_routed_experts)]), requires_grad=False)
-                moe_layer.w2 = nn.Parameter(torch.stack([moe_layer.experts[i].down_proj.weight.T for i in range(moe_layer.config.n_routed_experts)]), requires_grad=False)
-                moe_layer.w3 = nn.Parameter(torch.stack([moe_layer.experts[i].up_proj.weight.T for i in range(moe_layer.config.n_routed_experts)]), requires_grad=False)
                 del moe_layer.experts
                 gc.collect()
                 moe_layer.experts = None

             if isinstance(layer.mlp, DeepseekV2MoE):
                 moe_layer = layer.mlp
                 # combine experts
+                moe_layer.w1 = nn.Parameter(torch.stack([moe_layer.experts[i].gate_proj.weight.T for i in range(moe_layer.config.n_routed_experts)]))
+                moe_layer.w2 = nn.Parameter(torch.stack([moe_layer.experts[i].down_proj.weight.T for i in range(moe_layer.config.n_routed_experts)]))
+                moe_layer.w3 = nn.Parameter(torch.stack([moe_layer.experts[i].up_proj.weight.T for i in range(moe_layer.config.n_routed_experts)]))
                 del moe_layer.experts
                 gc.collect()
                 moe_layer.experts = None