Update app.py

app.py

@@ -11,13 +11,26 @@ headers = {
 }
 
 
-#import { HfInference } from '@huggingface/inference'
-#hf = new HfInference(API_URL, API_KEY)
-#prompt = 'Write instructions to teach anyone to write a discharge plan. List the entities, features and relationships to CCDA and FHIR objects in boldface.'
-#stream = hf.textGenerationStream({ inputs: prompt })
-#for await (const r of stream) { 
-#    st.markdown(r.token.text)
-#}
+from huggingface_hub import InferenceClient
+endpoint_url = API_URL
+hf_token = API_KEY
+client = InferenceClient(endpoint_url, token=hf_token)
+gen_kwargs = dict(
+    max_new_tokens=512,
+    top_k=30,
+    top_p=0.9,
+    temperature=0.2,
+    repetition_penalty=1.02,
+    stop_sequences=["\nUser:", "<|endoftext|>", "</s>"],
+)
+prompt = f"Write instructions to teach anyone to write a discharge plan. List the entities, features and relationships to CCDA and FHIR objects in boldface."
+stream = client.text_generation(prompt, stream=True, details=True, **gen_kwargs)
+for r in stream:
+    if r.token.special:
+        continue
+    if r.token.text in gen_kwargs["stop_sequences"]:
+        break
+    st.markdown(r.token.text, end = "")
 
 
 def query(payload):