Upload 2 files

app.py

@@ -19,6 +19,7 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
             gr.Markdown("""# Natural Text to SD Prompt Translator With LLM alpha
                             Text in natural language (English, Japanese, ...) => Prompt
                         """, elem_classes="title")
+            state = gr.State(value={})
             with gr.Group():
                 chatbot = gr.Chatbot(show_copy_button=True, show_share_button=False, layout="bubble", container=True)
                 with gr.Row():
@@ -98,22 +99,22 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
     gr.on(
         triggers=[chat_msg.submit, chat_submit.click],
         fn=dolphin_respond,
-        inputs=[chat_msg, chatbot, chat_model, chat_sysmsg, chat_tokens, chat_temperature, chat_topp, chat_topk, chat_rp],
+        inputs=[chat_msg, chatbot, chat_model, chat_sysmsg, chat_tokens, chat_temperature, chat_topp, chat_topk, chat_rp, state],
         outputs=[chatbot],
         queue=True,
         show_progress="full",
         trigger_mode="once",
-    ).success(dolphin_parse, [chatbot], [output_text, copy_btn, copy_btn_pony]).success(
+    ).success(dolphin_parse, [chatbot, state], [output_text, copy_btn, copy_btn_pony]).success(
         convert_danbooru_to_e621_prompt, [output_text, tag_type], [output_text_pony], queue=False,
     ).success(insert_recom_prompt, [output_text, dummy_np, recom_animagine], [output_text, dummy_np], queue=False,
     ).success(insert_recom_prompt, [output_text_pony, dummy_np_pony, recom_pony], [output_text_pony, dummy_np_pony], queue=False)
     chat_clear.click(lambda: None, None, chatbot, queue=False)
-    chat_model.change(select_dolphin_model, [chat_model], [chat_model, chat_format, chat_model_info], queue=True, show_progress="full")\
+    chat_model.change(select_dolphin_model, [chat_model, state], [chat_model, chat_format, chat_model_info, state], queue=True, show_progress="full")\
     .success(lambda: None, None, chatbot, queue=False)
-    chat_format.change(select_dolphin_format, [chat_format], [chat_format], queue=False)\
+    chat_format.change(select_dolphin_format, [chat_format, state], [chat_format, state], queue=False)\
     .success(lambda: None, None, chatbot, queue=False)
-    chat_mode.change(select_dolphin_sysprompt, [chat_mode], [chat_sysmsg], queue=False)
-    chat_lang.change(select_dolphin_language, [chat_lang], [chat_sysmsg], queue=False)
+    chat_mode.change(select_dolphin_sysprompt, [chat_mode, state], [chat_sysmsg, state], queue=False)
+    chat_lang.change(select_dolphin_language, [chat_lang, state], [chat_sysmsg, state], queue=False)
     gr.on(
         triggers=[chat_add_text.submit, chat_add_submit.click],
         fn=add_dolphin_models,
@@ -128,10 +129,8 @@ with gr.Blocks(theme='NoCrypt/miku@>=1.2.2', fill_width=True, css=css, delete_ca
 
     generate_image_btn.click(generate_image, [output_text, dummy_np], [result_image], show_progress="full")
 
-    pg_chat_model.change(select_dolphin_model, [pg_chat_model], [pg_chat_model, pg_chat_format, pg_chat_model_info], queue=True, show_progress="full")#\
-    #.success(lambda: None, None, pg_chatbot, queue=False)
-    pg_chat_format.change(select_dolphin_format, [pg_chat_format], [pg_chat_format], queue=False)#\
-    #.success(lambda: None, None, pg_chatbot, queue=False)
+    pg_chat_model.change(select_dolphin_model, [pg_chat_model, state], [pg_chat_model, pg_chat_format, pg_chat_model_info, state], queue=True, show_progress="full")
+    pg_chat_format.change(select_dolphin_format, [pg_chat_format, state], [pg_chat_format, state], queue=False)
     gr.on(
         triggers=[pg_chat_add_text.submit, pg_chat_add_submit.click],
         fn=add_dolphin_models,

llmdolphin.py

@@ -3,6 +3,7 @@ import gradio as gr
 from pathlib import Path
 import re
 import torch
+from typing import Any
 from huggingface_hub import hf_hub_download, HfApi
 from llama_cpp import Llama
 from llama_cpp_agent import LlamaCppAgent, MessagesFormatterType
@@ -75,6 +76,26 @@ llm_models = {
     "ChatWaifu_22B_v2.0_preview.Q4_K_S.gguf": ["mradermacher/ChatWaifu_22B_v2.0_preview-GGUF", MessagesFormatterType.MISTRAL],
     "ChatWaifu_v1.4.Q5_K_M.gguf": ["mradermacher/ChatWaifu_v1.4-GGUF", MessagesFormatterType.MISTRAL],
     "ChatWaifu_v1.3.1.Q4_K_M.gguf": ["mradermacher/ChatWaifu_v1.3.1-GGUF", MessagesFormatterType.MISTRAL],
+    "Flowable-Docs-Llama-3.1-8B.Q5_K_M.gguf": ["mradermacher/Flowable-Docs-Llama-3.1-8B-GGUF", MessagesFormatterType.LLAMA_3],
+    "slimorca-gemma2-9b-fft.Q4_K_M.gguf": ["mradermacher/slimorca-gemma2-9b-fft-GGUF", MessagesFormatterType.ALPACA],
+    "TQ2.5-14B-Sugarquill-v1-Q4_K_M.gguf": ["bartowski/TQ2.5-14B-Sugarquill-v1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "magnum-v3-9b-customgemma2.i1-Q4_K_M.gguf": ["mradermacher/magnum-v3-9b-customgemma2-i1-GGUF", MessagesFormatterType.ALPACA],
+    "Captain_BMO-12B.Q4_K_M.gguf": ["mradermacher/Captain_BMO-12B-GGUF", MessagesFormatterType.MISTRAL],
+    "LemonP-8B-Model_Stock.i1-Q5_K_M.gguf": ["mradermacher/LemonP-8B-Model_Stock-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "Ice0.31-08.11-RP.i1-Q5_K_M.gguf": ["mradermacher/Ice0.31-08.11-RP-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "EVA-Qwen2.5-14B-v0.2.i1-Q4_K_M.gguf": ["mradermacher/EVA-Qwen2.5-14B-v0.2-i1-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "L3.1-Dark-Planet-SpinFire-Uncensored-8B-D_AU-Q4_k_m.gguf": ["DavidAU/L3.1-Dark-Planet-SpinFire-Uncensored-8B-GGUF", MessagesFormatterType.LLAMA_3],
+    "EndlessRP-v2-7B.Q5_K_M.gguf": ["mradermacher/EndlessRP-v2-7B-GGUF", MessagesFormatterType.MISTRAL],
+    "badger-lambda-0-llama-3-8b.i1-Q5_K_M.gguf": ["mradermacher/badger-lambda-0-llama-3-8b-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "L3.1-Artemis-e2-8B.i1-Q5_K_M.gguf": ["mradermacher/L3.1-Artemis-e2-8B-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "MN-12B-Inferor-v0.0.Q4_K_M.gguf": ["mradermacher/MN-12B-Inferor-v0.0-GGUF", MessagesFormatterType.MISTRAL],
+    "Eclipse-13B-dpo.i1-Q4_K_M.gguf": ["mradermacher/Eclipse-13B-dpo-i1-GGUF", MessagesFormatterType.MISTRAL],
+    "Epic_Fiction-8b-v0.1.Q5_K_M.gguf": ["mradermacher/Epic_Fiction-8b-v0.1-GGUF", MessagesFormatterType.MISTRAL],
+    "Llama-3-8B-StoryGenerator.i1-Q5_K_M.gguf": ["mradermacher/Llama-3-8B-StoryGenerator-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "badger-mu-llama-3-8b.i1-Q5_K_M.gguf": ["mradermacher/badger-mu-llama-3-8b-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "badger-writer-llama-3-8b.i1-Q5_K_M.gguf": ["mradermacher/badger-writer-llama-3-8b-i1-GGUF", MessagesFormatterType.LLAMA_3],
+    "Qwen-Rui-SE.Q5_K_M.gguf": ["mradermacher/Qwen-Rui-SE-GGUF", MessagesFormatterType.OPEN_CHAT],
+    "Valor-7B-v0.1.i1-Q4_K_M.gguf": ["mradermacher/Valor-7B-v0.1-i1-GGUF", MessagesFormatterType.MISTRAL],
     "Fraken-Maid-TW-Slerp.i1-Q5_K_M.gguf": ["mradermacher/Fraken-Maid-TW-Slerp-i1-GGUF", MessagesFormatterType.MISTRAL],
     "KunoichiLake-2x7b.Q4_K_M.gguf": ["mradermacher/KunoichiLake-2x7b-GGUF", MessagesFormatterType.MISTRAL],
     "Llama-3.1-SuperNova-8B-Lite_TIES_with_Base-Q4_K_M.gguf": ["bartowski/Llama-3.1-SuperNova-8B-Lite_TIES_with_Base-GGUF", MessagesFormatterType.LLAMA_3],
@@ -908,25 +929,35 @@ llm_formats = {
 llm_languages = ["English", "Japanese", "Chinese", "Korean", "Spanish", "Portuguese", "German", "French", "Finnish", "Russian"]
 llm_models_tupled_list = []
 default_llm_model_filename = list(llm_models.keys())[0]
-override_llm_format = None
 device = "cuda" if torch.cuda.is_available() else "cpu"
 
 
-def to_list(s):
+def to_list(s: str):
     return [x.strip() for x in s.split(",") if not s == ""]
 
 
-def list_uniq(l):
+def list_uniq(l: list):
     return sorted(set(l), key=l.index)
 
 
+def get_state(state: dict, key: str):
+    if key in state.keys(): return state[key]
+    else:
+        print(f"State '{key}' not found.")
+        return None
+
+
+def set_state(state: dict, key: str, value: Any):
+    state[key] = value
+
+
 @wrapt_timeout_decorator.timeout(dec_timeout=3.5)
-def to_list_ja(s):
+def to_list_ja(s: str):
     s = re.sub(r'[、。]', ',', s)
     return [x.strip() for x in s.split(",") if not s == ""]
 
 
-def is_japanese(s):
+def is_japanese(s: str):
     import unicodedata
     for ch in s:
         name = unicodedata.name(ch, "") 
@@ -964,7 +995,7 @@ def download_llm_models():
         llm_models_tupled_list.append((name, value))
 
 
-def download_llm_model(filename):
+def download_llm_model(filename: str):
     if not filename in llm_models.keys(): return default_llm_model_filename
     try:
         hf_hub_download(repo_id = llm_models[filename][0], filename = filename, local_dir = llm_models_dir)
@@ -975,7 +1006,7 @@ def download_llm_model(filename):
     return filename
 
 
-def get_dolphin_model_info(filename):
+def get_dolphin_model_info(filename: str):
     md = "None"
     items = llm_models.get(filename, None)
     if items:
@@ -983,20 +1014,18 @@ def get_dolphin_model_info(filename):
     return md
 
 
-def select_dolphin_model(filename, progress=gr.Progress(track_tqdm=True)):
-    global override_llm_format
-    override_llm_format = None
+def select_dolphin_model(filename: str, state: dict, progress=gr.Progress(track_tqdm=True)):
+    set_state(state, "override_llm_format", None)
     progress(0, desc="Loading model...")
     value = download_llm_model(filename)
     progress(1, desc="Model loaded.")
     md = get_dolphin_model_info(filename)
-    return gr.update(value=value, choices=get_dolphin_models()), gr.update(value=get_dolphin_model_format(value)), gr.update(value=md)
+    return gr.update(value=value, choices=get_dolphin_models()), gr.update(value=get_dolphin_model_format(value)), gr.update(value=md), state
 
 
-def select_dolphin_format(format_name):
-    global override_llm_format
-    override_llm_format = llm_formats[format_name]
-    return gr.update(value=format_name)
+def select_dolphin_format(format_name: str, state: dict):
+    set_state(state, "override_llm_format", llm_formats[format_name])
+    return gr.update(value=format_name), state
 
 
 download_llm_model(default_llm_model_filename)
@@ -1017,14 +1046,14 @@ def get_key_from_value(d, val):
     return None
 
 
-def get_dolphin_model_format(filename):
+def get_dolphin_model_format(filename: str):
     if not filename in llm_models.keys(): filename = default_llm_model_filename
     format = llm_models[filename][1]
     format_name = get_key_from_value(llm_formats, format)
     return format_name
 
 
-def add_dolphin_models(query, format_name):
+def add_dolphin_models(query: str, format_name: str):
     global llm_models
     api = HfApi()
     add_models = {}
@@ -1055,8 +1084,6 @@ def add_dolphin_models(query, format_name):
     return gr.update(choices=choices, value=choices[-1][1])
 
 
-dolphin_output_language = "English"
-dolphin_sysprompt_mode = "Default"
 dolphin_system_prompt = {"Default": r'''You are a helpful AI assistant to generate messages for AI that outputs an image when I enter a message.
 The message must have the following [Tags] generated in strict accordance with the following [Rules]:
 ```
@@ -1247,8 +1274,11 @@ Output should be enclosed in //GENBEGIN//:// and //://GENEND//. The text to be g
 "Chat with LLM": r"You are a helpful AI assistant. Respond in <LANGUAGE>."}
 
 
-def get_dolphin_sysprompt():
-    prompt = re.sub('<LANGUAGE>', dolphin_output_language, dolphin_system_prompt.get(dolphin_sysprompt_mode, ""))
+def get_dolphin_sysprompt(state: dict={}):
+    dolphin_sysprompt_mode = get_state(state, "dolphin_sysprompt_mode")
+    dolphin_output_language = get_state(state, "dolphin_output_language")
+    prompt = re.sub('<LANGUAGE>', dolphin_output_language if dolphin_output_language else llm_languages[0],
+                    dolphin_system_prompt.get(dolphin_sysprompt_mode, list(dolphin_system_prompt.keys())[0]))
     return prompt
 
 
@@ -1256,23 +1286,23 @@ def get_dolphin_sysprompt_mode():
     return list(dolphin_system_prompt.keys())
 
 
-def select_dolphin_sysprompt(key: str):
-    global dolphin_sysprompt_mode
+def select_dolphin_sysprompt(key: str, state: dict):
+    dolphin_sysprompt_mode = get_state(state, "dolphin_sysprompt_mode")
     if not key in dolphin_system_prompt.keys():
         dolphin_sysprompt_mode = "Default"
     else:
         dolphin_sysprompt_mode = key
-    return gr.update(value=get_dolphin_sysprompt())
+    set_state(state, "dolphin_sysprompt_mode", dolphin_sysprompt_mode)
+    return gr.update(value=get_dolphin_sysprompt(state)), state
 
 
 def get_dolphin_languages():
     return llm_languages
 
 
-def select_dolphin_language(lang: str):
-    global dolphin_output_language
-    dolphin_output_language = lang
-    return gr.update(value=get_dolphin_sysprompt())
+def select_dolphin_language(lang: str, state: dict):
+    set_state(state, "dolphin_output_language", lang)
+    return gr.update(value=get_dolphin_sysprompt()), state
 
 
 @wrapt_timeout_decorator.timeout(dec_timeout=5.0)
@@ -1293,11 +1323,12 @@ def dolphin_respond(
     top_p: float = 0.95,
     top_k: int = 40,
     repeat_penalty: float = 1.1,
+    state: dict = {},
     progress=gr.Progress(track_tqdm=True),
 ):
     try:
         progress(0, desc="Processing...")
-
+        override_llm_format = get_state(state, "override_llm_format")
         if override_llm_format:
             chat_template = override_llm_format
         else:
@@ -1363,8 +1394,10 @@ def dolphin_respond(
 
 def dolphin_parse(
     history: list[tuple[str, str]],
+    state: dict,
 ):
     try:
+        dolphin_sysprompt_mode = get_state(state, "dolphin_sysprompt_mode")
         if dolphin_sysprompt_mode == "Chat with LLM" or not history or len(history) < 1:
             return "", gr.update(), gr.update()
         msg = history[-1][0]
@@ -1392,12 +1425,14 @@ def dolphin_respond_auto(
     top_p: float = 0.95,
     top_k: int = 40,
     repeat_penalty: float = 1.1,
+    state: dict = {},
     progress=gr.Progress(track_tqdm=True),
 ):
     try:
         #if not is_japanese(message): return [(None, None)]
         progress(0, desc="Processing...")
 
+        override_llm_format = get_state(state, "override_llm_format")
         if override_llm_format:
             chat_template = override_llm_format
         else:
@@ -1465,9 +1500,11 @@ def dolphin_respond_auto(
 def dolphin_parse_simple(
     message: str,
     history: list[tuple[str, str]],
+    state: dict,
 ):
     try:
         #if not is_japanese(message): return message
+        dolphin_sysprompt_mode = get_state(state, "dolphin_sysprompt_mode")
         if dolphin_sysprompt_mode == "Chat with LLM" or not history or len(history) < 1: return message
         msg = history[-1][0]
         raw_prompt = get_raw_prompt(msg)
@@ -1499,8 +1536,10 @@ def respond_playground(
     top_p,
     top_k,
     repeat_penalty,
+    state,
 ):
     try:
+        override_llm_format = get_state(state, "override_llm_format")
         if override_llm_format:
             chat_template = override_llm_format
         else: