app.py

import os
os.system("pip uninstall -y gradio")
os.system("pip install gradio==3.43.0")

from lmdeploy.serve.gradio.turbomind_coupled import run_local
from lmdeploy.messages import TurbomindEngineConfig

backend_config = TurbomindEngineConfig(max_batch_size=1, cache_max_entry_count=0.05)
model_path = 'yentinglin/Llama-3-Taiwan-8B-Instruct'
run_local(model_path, backend_config=backend_config, server_name="huggingface-space")