Update app.py

app.py

@@ -95,37 +95,43 @@ def add_witty_humor_buttons():
 
 # Function to Stream Inference Client for Inference Endpoint Responses
 def StreamLLMChatResponse(prompt):
-    endpoint_url = API_URL
-    hf_token = API_KEY
-    client = InferenceClient(endpoint_url, token=hf_token)
-    gen_kwargs = dict(
-        max_new_tokens=512,
-        top_k=30,
-        top_p=0.9,
-        temperature=0.2,
-        repetition_penalty=1.02,
-        stop_sequences=["\nUser:", "<|endoftext|>", "</s>"],
-    )
-    stream = client.text_generation(prompt, stream=True, details=True, **gen_kwargs)
-    report=[]
-    res_box = st.empty()
-    collected_chunks=[]
-    collected_messages=[]
-    for r in stream:
-        if r.token.special:
-            continue
-        if r.token.text in gen_kwargs["stop_sequences"]:
-            break
-        collected_chunks.append(r.token.text)
-        chunk_message = r.token.text
-        collected_messages.append(chunk_message)
-        try:
-            report.append(r.token.text)
-            if len(r.token.text) > 0:
-                result="".join(report).strip()
-                res_box.markdown(f'*{result}*')
-        except:
-            st.write(' ')
+
+    try:
+        endpoint_url = API_URL
+        hf_token = API_KEY
+        client = InferenceClient(endpoint_url, token=hf_token)
+        gen_kwargs = dict(
+            max_new_tokens=512,
+            top_k=30,
+            top_p=0.9,
+            temperature=0.2,
+            repetition_penalty=1.02,
+            stop_sequences=["\nUser:", "<|endoftext|>", "</s>"],
+        )
+        stream = client.text_generation(prompt, stream=True, details=True, **gen_kwargs)
+        report=[]
+        res_box = st.empty()
+        collected_chunks=[]
+        collected_messages=[]
+        for r in stream:
+            if r.token.special:
+                continue
+            if r.token.text in gen_kwargs["stop_sequences"]:
+                break
+            collected_chunks.append(r.token.text)
+            chunk_message = r.token.text
+            collected_messages.append(chunk_message)
+            try:
+                report.append(r.token.text)
+                if len(r.token.text) > 0:
+                    result="".join(report).strip()
+                    res_box.markdown(f'*{result}*')
+            except:
+                st.write(' ')
+    except:
+        st.write('DromeLlama is asleep. Starting up now on A10 - please give 5 minutes then retry as KEDA scales up from zero to activate running container(s).')
+
+
 
 def query(payload):
     response = requests.post(API_URL, headers=headers, json=payload)