app file

app.py

@@ -0,0 +1,23 @@
+from peft import PeftModel, PeftConfig
+from transformers import AutoModelForCausalLM, AutoTokenizer
+import gradio as gr
+
+config = PeftConfig.from_pretrained("pranjalpandey/gemma-open-instruct")
+model = AutoModelForCausalLM.from_pretrained("google/gemma-2b")
+model = PeftModel.from_pretrained(model, "pranjalpandey/gemma-open-instruct")
+# model = AutoPeftModelForCausalLM.from_pretrained("pranjalpandey/llama-7b-finetuned-dialogue-summarizer")
+tokenizer = AutoTokenizer.from_pretrained("pranjalpandey/gemma-open-instruct")
+model = model.to("cuda")
+
+def response(prompt):
+  inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
+  outputs = model.generate(input_ids=inputs["input_ids"], max_new_tokens=100)
+  return tokenizer.batch_decode(outputs.detach().cpu().numpy(), skip_special_tokens=True)[0].split("# Response:")[1].strip()
+
+ir = gr.Interface(
+    fn=response,
+    inputs=["text"],
+    outputs=["text"],
+)
+
+ir.launch()

requirements.txt

@@ -0,0 +1,4 @@
+peft 
+accelerate 
+transformers
+gradio