new_llm

analyzer.py

@@ -8,7 +8,8 @@ def analyze_code(code: str) -> str:
     Returns the analysis as a string.
     """
     from openai import OpenAI
-    client = OpenAI()
+    client = OpenAI(api_key=os.getenv("modal_api"))
+    client.base_url = os.getenv("base_url")
     system_prompt = (
         "You are a helpful assistant. Analyze the code given to you. "
         "Return your response strictly in JSON format with the following keys: "
@@ -23,7 +24,7 @@ def analyze_code(code: str) -> str:
         "}"
     )
     response = client.chat.completions.create(
-        model="gpt-4o-mini",  # GPT-4.1 mini
+        model="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",  # Updated model
         messages=[
             {"role": "system", "content": system_prompt},
             {"role": "user", "content": code}

chatbot_page.py

@@ -1,4 +1,5 @@
 import gradio as gr
+import os
 # from analyzer import analyze_code
 
 # System prompt for the chatbot
@@ -15,7 +16,8 @@ conversation_history = []
 # Function to handle chat
 def chat_with_user(user_message, history):
     from openai import OpenAI
-    client = OpenAI()
+    client = OpenAI(api_key=os.getenv("modal_api"))
+    client.base_url = os.getenv("base_url")
     # Build the message list for the LLM
     messages = [
         {"role": "system", "content": CHATBOT_SYSTEM_PROMPT}
@@ -26,7 +28,7 @@ def chat_with_user(user_message, history):
             messages.append({"role": "assistant", "content": msg[1]})
     messages.append({"role": "user", "content": user_message})
     response = client.chat.completions.create(
-        model="gpt-4o-mini",
+        model="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",
         messages=messages,
         max_tokens=256,
         temperature=0.7
@@ -38,7 +40,8 @@ def chat_with_user(user_message, history):
 def extract_keywords_from_conversation(history):
     print("Extracting keywords from conversation...")
     from openai import OpenAI
-    client = OpenAI()
+    client = OpenAI(api_key=os.getenv("modal_api"))
+    client.base_url = os.getenv("base_url")
     # Combine all user and assistant messages into a single string
     conversation = "\n".join([f"User: {msg[0]}\nAssistant: {msg[1]}" for msg in history if msg[1]])
     system_prompt = (
@@ -50,7 +53,7 @@ def extract_keywords_from_conversation(history):
         "Conversation:\n" + conversation + "\n\nExtract about 5 keywords for Hugging Face repo search."
     )
     response = client.chat.completions.create(
-        model="gpt-4o-mini",
+        model="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",
         messages=[
             {"role": "system", "content": system_prompt},
             {"role": "user", "content": user_prompt}

test.py

@@ -0,0 +1,23 @@
+"""This simple script shows how to interact with an OpenAI-compatible server from a client."""
+
+# import argparse
+
+# import modal
+from openai import OpenAI
+import os
+
+client = OpenAI(api_key=os.getenv("modal_api"))
+client.base_url = (
+    "https://alexprincecursor--example-vllm-openai-compatible-serve.modal.run/v1/"
+)
+
+response = client.chat.completions.create(
+        model="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",  # GPT-4.1 mini
+        messages=[
+            {"role": "system", "content": "You are a rockstar lyric generator. You are given a song and you need to generate a lyric for it."},
+            {"role": "user", "content":"The song is 'Bohemian Rhapsody' by Queen."}
+        ],
+        max_tokens=512,
+        temperature=0.7
+    )
+print(response.choices[0].message.content)