Create app.py

app.py

@@ -0,0 +1,115 @@
+import gradio as gr
+from text_generation import Client
+# text-generation 0.6.0
+
+eos_token = "</s>"
+
+def _concat_messages(messages):
+    message_text = ""
+    for message in messages:
+        if message["role"] == "system":
+            message_text += "<|system|>\n" + message["content"].strip() + "\n"
+        elif message["role"] == "user":
+            message_text += "<|user|>\n" + message["content"].strip() + "\n"
+        elif message["role"] == "assistant":
+            message_text += "<|assistant|>\n" + message["content"].strip() + eos_token + "\n"
+        else:
+            raise ValueError("Invalid role: {}".format(message["role"]))
+    return message_text
+
+endpoint_url = "http://ec2-52-193-118-191.ap-northeast-1.compute.amazonaws.com:8080"
+client = Client(endpoint_url, timeout=120)
+
+def generate_response(user_input, max_new_token: 100, top_p, temperature, top_k, do_sample, repetition_penalty):
+    msg = _concat_messages([
+        {"role": "system", "content": "你是一個由國立台灣大學的NLP實驗室開發的大型語言模型。你基於Transformer架構被訓練，並已經經過大量的台灣中文語料庫的訓練。你的設計目標是理解和生成優雅的繁體中文，並具有跨語境和跨領域的對話能力。使用者可以向你提問任何問題或提出任何話題，並期待從你那裡得到高質量的回答。你應該要盡量幫助使用者解決問題，提供他們需要的資訊，並在適當時候給予建議。"},
+        {"role": "user", "content": user_input},
+    ])
+    msg += "<|assistant|>\n"
+    res = client.generate(msg, stop_sequences=["<|assistant|>", eos_token, "<|system|>", "<|user|>"],
+                          max_new_tokens=1000)
+    return [("assistant", res.generated_text)]
+
+with gr.Blocks() as demo:
+    # github_banner_path = 'https://raw.githubusercontent.com/ymcui/Chinese-LLaMA-Alpaca/main/pics/banner.png'
+    # gr.HTML(f'<p align="center"><a href="https://github.com/ymcui/Chinese-LLaMA-Alpaca"><img src={github_banner_path} width="700"/></a></p>')
+    # gr.Markdown("> 为了促进大模型在中文NLP社区的开放研究，本项目开源了中文LLaMA模型和指令精调的Alpaca大模型。这些模型在原版LLaMA的基础上扩充了中文词表并使用了中文数据进行二次预训练，进一步提升了中文基础语义理解能力。同时，中文Alpaca模型进一步使用了中文指令数据进行精调，显著提升了模型对指令的理解和执行能力。")
+    chatbot = gr.Chatbot()
+    with gr.Row():
+        with gr.Column(scale=4):
+            with gr.Column(scale=12):
+                user_input = gr.Textbox(
+                    show_label=False,
+                    placeholder="Shift + Enter发送消息...",
+                    lines=10).style(
+                    container=False)
+            with gr.Column(min_width=32, scale=1):
+                submitBtn = gr.Button("Submit", variant="primary")
+        with gr.Column(scale=1):
+            emptyBtn = gr.Button("Clear History")
+            max_new_token = gr.Slider(
+                0,
+                4096,
+                value=512,
+                step=1.0,
+                label="Maximum New Token Length",
+                interactive=True)
+            top_p = gr.Slider(0, 1, value=0.9, step=0.01,
+                              label="Top P", interactive=True)
+            temperature = gr.Slider(
+                0,
+                1,
+                value=0.5,
+                step=0.01,
+                label="Temperature",
+                interactive=True)
+            top_k = gr.Slider(1, 40, value=40, step=1,
+                              label="Top K", interactive=True)
+            do_sample = gr.Checkbox(
+                value=True,
+                label="Do Sample",
+                info="use random sample strategy",
+                interactive=True)
+            repetition_penalty = gr.Slider(
+                1.0,
+                3.0,
+                value=1.1,
+                step=0.1,
+                label="Repetition Penalty",
+                interactive=True)
+
+    params = [user_input, chatbot]
+    predict_params = [
+        chatbot,
+        max_new_token,
+        top_p,
+        temperature,
+        top_k,
+        do_sample,
+        repetition_penalty]
+
+    submitBtn.click(
+        generate_response,
+        [user_input],
+        [chatbot],
+        queue=False).then(
+        None,
+        None,
+        [user_input],
+        queue=False)
+
+    user_input.submit(
+        generate_response,
+        [user_input],
+        [chatbot],
+        queue=False).then(
+        None,
+        None,
+        [user_input],
+        queue=False)
+
+    submitBtn.click(lambda: None, [], [user_input])
+
+    emptyBtn.click(lambda: chatbot.reset(), outputs=[chatbot], show_progress=True)
+
+demo.launch(share=True)