Create app.py

app.py

@@ -0,0 +1,53 @@
+# encoding = "utf-8"
+
+
+'''
+This is a mediator: a gradio server for OpenAI APIs
+'''
+
+import os
+import json
+import argparse
+import gradio as gr
+import requests
+from openai import OpenAI
+
+def http_bot(messages, argsbox):
+
+    args = json.loads(argsbox)
+    messages = json.loads(messages)
+
+    print(messages)
+    print(argsbox)
+
+    client = OpenAI(api_key=args["api_key"], base_url = args["base_url"])
+    # n = 0
+    # while True:
+        # try:
+    chat_completion = client.chat.completions.create(
+        messages=messages,
+        model=args["model"], #"gpt-3.5-turbo-16k",  # "gpt-3.5-turbo", # gpt-4-1106-preview
+        temperature=float(args["temperature"]),
+        max_tokens=int(args["max_tokens"])
+    )
+        #     break
+        # except Exception as e:
+        #     continue
+    print(chat_completion)
+    return chat_completion.choices[0].message.content
+
+
+
+with gr.Blocks() as demo:
+    gr.Markdown("# vLLM text completion demo\n")
+    inputbox = gr.Textbox(label="Input",
+                          placeholder="Enter text and press ENTER")
+    argsbox = gr.Textbox(label="Args", placeholder="a dict of {api_key, base_url, model, temperature, max_tokens}")
+    outputbox = gr.Textbox(label="Output",
+                           placeholder="Generated result from the model")
+    submit = gr.Button("Submit")
+
+    submit.click(http_bot, [inputbox, argsbox], [outputbox], api_name="submit")
+
+demo.launch(share=True)
+