import ast import enum import importlib import json import os import pickle import subprocess import tempfile import traceback import zipfile from typing import Any, ClassVar from urllib.parse import urljoin import pandas as pd import requests import tqdm # Add tqdm for progress bar from langchain_core.callbacks import BaseCallbackHandler from langchain_core.messages.base import get_msg_title_repr from langchain_core.tools import StructuredTool from langchain_core.utils.interactive_env import is_interactive_env from pydantic import BaseModel, Field, ValidationError def run_bash_script(script: str) -> str: """Run a Bash script using subprocess. Args: script: Bash script to run Returns: Output of the Bash script Example: ``` # Example of a complex Bash script script = ''' #!/bin/bash # Define variables DATA_DIR="/path/to/data" OUTPUT_FILE="results.txt" # Create output directory if it doesn't exist mkdir -p $(dirname $OUTPUT_FILE) # Loop through files for file in $DATA_DIR/*.txt; do echo "Processing $file..." # Count lines in each file line_count=$(wc -l < $file) echo "$file: $line_count lines" >> $OUTPUT_FILE done echo "Processing complete. Results saved to $OUTPUT_FILE" ''' result = run_bash_script(script) print(result) ``` """ try: # Trim any leading/trailing whitespace script = script.strip() # If the script is empty, return an error if not script: return "Error: Empty script" # Create a temporary file to store the Bash script with tempfile.NamedTemporaryFile(suffix=".sh", mode="w", delete=False) as f: # Add shebang if not present if not script.startswith("#!/"): f.write("#!/bin/bash\n") # Add set -e to exit on error if "set -e" not in script: f.write("set -e\n") f.write(script) temp_file = f.name # Make the script executable os.chmod(temp_file, 0o755) # Get current environment variables and working directory env = os.environ.copy() cwd = os.getcwd() # Run the Bash script with the current environment and working directory result = subprocess.run( [temp_file], shell=True, capture_output=True, text=True, check=False, env=env, cwd=cwd, ) # Clean up the temporary file os.unlink(temp_file) # Return the output if result.returncode != 0: traceback.print_stack() print(result) return f"Error running Bash script (exit code {result.returncode}):\n{result.stderr}" else: return result.stdout except Exception as e: traceback.print_exc() return f"Error running Bash script: {str(e)}" # Keep the run_cli_command for backward compatibility def run_cli_command(command: str) -> str: """Run a CLI command using subprocess. Args: command: CLI command to run Returns: Output of the CLI command """ try: # Trim any leading/trailing whitespace command = command.strip() # If the command is empty, return an error if not command: return "Error: Empty command" # Split the command into a list of arguments, handling quoted arguments correctly import shlex args = shlex.split(command) # Run the command result = subprocess.run(args, capture_output=True, text=True, check=False) # Return the output if result.returncode != 0: return f"Error running command '{command}':\n{result.stderr}" else: return result.stdout except Exception as e: return f"Error running command '{command}': {str(e)}" def run_with_timeout(func, args=None, kwargs=None, timeout=600): """Run a function with a timeout using threading instead of multiprocessing. This allows variables to persist in the global namespace between function calls. Returns the function result or a timeout error message. """ if args is None: args = [] if kwargs is None: kwargs = {} import ctypes import queue import threading result_queue = queue.Queue() def thread_func(func, args, kwargs, result_queue): """Function to run in a separate thread.""" try: result = func(*args, **kwargs) result_queue.put(("success", result)) except Exception as e: result_queue.put(("error", str(e))) # Start a separate thread thread = threading.Thread(target=thread_func, args=(func, args, kwargs, result_queue)) thread.daemon = True # Set as daemon so it will be killed when main thread exits thread.start() # Wait for the specified timeout thread.join(timeout) # Check if the thread is still running after timeout if thread.is_alive(): print(f"TIMEOUT: Code execution timed out after {timeout} seconds") # Unfortunately, there's no clean way to force terminate a thread in Python # The recommended approach is to use daemon threads and let them be killed when main thread exits # Here, we'll try to raise an exception in the thread to make it stop try: # Get thread ID and try to terminate it thread_id = thread.ident if thread_id: # This is a bit dangerous and not 100% reliable # It attempts to raise a SystemExit exception in the thread res = ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(thread_id), ctypes.py_object(SystemExit)) if res > 1: # Oops, we raised too many exceptions ctypes.pythonapi.PyThreadState_SetAsyncExc(ctypes.c_long(thread_id), None) except Exception as e: print(f"Error trying to terminate thread: {e}") return f"ERROR: Code execution timed out after {timeout} seconds. Please try with simpler inputs or break your task into smaller steps." # Get the result from the queue if available try: status, result = result_queue.get(block=False) return result if status == "success" else f"Error in execution: {result}" except queue.Empty: return "Error: Execution completed but no result was returned" class api_schema(BaseModel): """api schema specification.""" api_schema: str | None = Field(description="The api schema as a dictionary") def function_to_api_schema(function_string, llm): prompt = """ Based on a code snippet and help me write an API docstring in the format like this: {{'name': 'get_gene_set_enrichment', 'description': 'Given a list of genes, identify a pathway that is enriched for this gene set. Return a list of pathway name, p-value, z-scores.', 'required_parameters': [{{'name': 'genes', 'type': 'List[str]', 'description': 'List of g`ene symbols to analyze', 'default': None}}], 'optional_parameters': [{{'name': 'top_k', 'type': 'int', 'description': 'Top K pathways to return', 'default': 10}}, {{'name': 'database', 'type': 'str', 'description': 'Name of the database to use for enrichment analysis', 'default': "gene_ontology"}}]}} Strictly follow the input from the function - don't create fake optional parameters. For variable without default values, set them as None, not null. For variable with boolean values, use capitalized True or False, not true or false. Do not add any return type in the docstring. Be as clear and succint as possible for the descriptions. Please do not make it overly verbose. Here is the code snippet: {code} """ llm = llm.with_structured_output(api_schema) for _ in range(7): try: api = llm.invoke(prompt.format(code=function_string)).dict()["api_schema"] return ast.literal_eval(api) # -> prefer "default": None # return json.loads(api) # -> prefer "default": null except Exception as e: print("API string:", api) print("Error parsing the API string:", e) continue return "Error: Could not parse the API schema" # return def get_all_functions_from_file(file_path): with open(file_path) as file: file_content = file.read() # Parse the file content into an AST (Abstract Syntax Tree) tree = ast.parse(file_content) # List to hold the top-level functions as strings functions = [] # Walk through the AST nodes for node in tree.body: # Only consider top-level nodes in the body if isinstance(node, ast.FunctionDef): # Check if the node is a function definition # Skip if function name starts with underscore if node.name.startswith("_"): continue start_line = node.lineno - 1 # Get the starting line of the function end_line = node.end_lineno # Get the ending line of the function (only available in Python 3.8+) func_code = file_content.splitlines()[start_line:end_line] functions.append("\n".join(func_code)) # Join lines of the function and add to the list return functions def write_python_code(request: str): from langchain_anthropic import ChatAnthropic from langchain_core.output_parsers import StrOutputParser from langchain_core.prompts import ChatPromptTemplate model = ChatAnthropic(model="claude-3-5-sonnet-20240620") template = """Write some python code to solve the user's problem. Return only python code in Markdown format, e.g.: ```python .... ```""" prompt = ChatPromptTemplate.from_messages([("system", template), ("human", "{input}")]) def _sanitize_output(text: str): _, after = text.split("```python") return after.split("```")[0] chain = prompt | model | StrOutputParser() | _sanitize_output return chain.invoke({"input": "write a code that " + request}) def execute_graphql_query( query: str, variables: dict, api_address: str = "https://api.genetics.opentargets.org/graphql", ) -> dict: """Executes a GraphQL query with variables and returns the data as a dictionary.""" headers = {"Content-Type": "application/json"} response = requests.post(api_address, json={"query": query, "variables": variables}, headers=headers) if response.status_code == 200: return response.json() else: print(response.text) response.raise_for_status() def get_tool_decorated_functions(relative_path): import ast import importlib.util import os # Get the directory of the current file (__init__.py) current_dir = os.path.dirname(os.path.abspath(__file__)) # Construct the absolute path from the relative path file_path = os.path.join(current_dir, relative_path) with open(file_path) as file: tree = ast.parse(file.read(), filename=file_path) tool_function_names = [] for node in ast.walk(tree): if isinstance(node, ast.FunctionDef): for decorator in node.decorator_list: if ( isinstance(decorator, ast.Name) and decorator.id == "tool" or ( isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Name) and decorator.func.id == "tool" ) ): tool_function_names.append(node.name) # Calculate the module name from the relative path package_path = os.path.relpath(file_path, start=current_dir) module_name = package_path.replace(os.path.sep, ".").rsplit(".", 1)[0] # Import the module and get the function objects spec = importlib.util.spec_from_file_location(module_name, file_path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) tool_functions = [getattr(module, name) for name in tool_function_names] return tool_functions def load_pickle(file): import pickle with open(file, "rb") as f: return pickle.load(f) def pretty_print(message, printout=True): if isinstance(message, tuple): title = message elif isinstance(message.content, list): title = get_msg_title_repr(message.type.title().upper() + " Message", bold=is_interactive_env()) if message.name is not None: title += f"\nName: {message.name}" for i in message.content: if i["type"] == "text": title += f"\n{i['text']}\n" elif i["type"] == "tool_use": title += f"\nTool: {i['name']}" title += f"\nInput: {i['input']}" if printout: print(f"{title}") else: title = get_msg_title_repr(message.type.title() + " Message", bold=is_interactive_env()) if message.name is not None: title += f"\nName: {message.name}" title += f"\n\n{message.content}" if printout: print(f"{title}") return title class CustomBaseModel(BaseModel): api_schema: ClassVar[dict] = None # Class variable to store api_schema # Add model_config with arbitrary_types_allowed=True model_config = {"arbitrary_types_allowed": True} @classmethod def set_api_schema(cls, schema: dict): cls.api_schema = schema @classmethod def model_validate(cls, obj): try: return super().model_validate(obj) except (ValidationError, AttributeError) as e: if not cls.api_schema: raise e # If no api_schema is set, raise original error error_msg = "Required Parameters:\n" for param in cls.api_schema["required_parameters"]: error_msg += f"- {param['name']} ({param['type']}): {param['description']}\n" error_msg += "\nErrors:\n" for err in e.errors(): field = err["loc"][0] if err["loc"] else "input" error_msg += f"- {field}: {err['msg']}\n" if not obj: error_msg += "\nNo input provided" else: error_msg += "\nProvided Input:\n" for key, value in obj.items(): error_msg += f"- {key}: {value}\n" missing_params = {param["name"] for param in cls.api_schema["required_parameters"]} - set(obj.keys()) if missing_params: error_msg += "\nMissing Parameters:\n" for param in missing_params: error_msg += f"- {param}\n" # # Create proper validation error structure raise ValidationError.from_exception_data( title="Validation Error", line_errors=[ { "type": "value_error", "loc": ("input",), "input": obj, "ctx": { "error": error_msg, }, } ], ) from None def safe_execute_decorator(func): def wrapper(*args, **kwargs): try: return func(*args, **kwargs) except Exception as e: return str(e) return wrapper def api_schema_to_langchain_tool(api_schema, mode="generated_tool", module_name=None): if mode == "generated_tool": module = importlib.import_module("histopath.tool.generated_tool." + api_schema["tool_name"] + ".api") elif mode == "custom_tool": module = importlib.import_module(module_name) api_function = getattr(module, api_schema["name"]) api_function = safe_execute_decorator(api_function) # Define a mapping from string type names to actual Python type objects type_mapping = { "string": str, "integer": int, "boolean": bool, "pandas": pd.DataFrame, # Use the imported pandas.DataFrame directly "str": str, "int": int, "bool": bool, "List[str]": list[str], "List[int]": list[int], "Dict": dict, "Any": Any, } # Create the fields and annotations annotations = {} for param in api_schema["required_parameters"]: param_type = param["type"] if param_type in type_mapping: annotations[param["name"]] = type_mapping[param_type] else: # For types not in the mapping, try a safer approach than direct eval try: annotations[param["name"]] = eval(param_type) except (NameError, SyntaxError): # Default to Any for unknown types annotations[param["name"]] = Any fields = {param["name"]: Field(description=param["description"]) for param in api_schema["required_parameters"]} # Create the ApiInput class dynamically ApiInput = type("Input", (CustomBaseModel,), {"__annotations__": annotations, **fields}) # Set the api_schema ApiInput.set_api_schema(api_schema) # Create the StructuredTool api_tool = StructuredTool.from_function( func=api_function, name=api_schema["name"], description=api_schema["description"], args_schema=ApiInput, return_direct=True, ) return api_tool class ID(enum.Enum): ENTREZ = "Entrez" ENSEMBL = "Ensembl without version" # e.g. ENSG00000123374 ENSEMBL_W_VERSION = "Ensembl with version" # e.g. ENSG00000123374.10 (needed for GTEx) def save_pkl(f, filename): with open(filename, "wb") as file: pickle.dump(f, file) def load_pkl(filename): with open(filename, "rb") as file: return pickle.load(file) _TEXT_COLOR_MAPPING = { "blue": "36;1", "yellow": "33;1", "pink": "38;5;200", "green": "32;1", "red": "31;1", } def color_print(text, color="blue"): color_str = _TEXT_COLOR_MAPPING[color] print(f"\u001b[{color_str}m\033[1;3m{text}\u001b[0m") class PromptLogger(BaseCallbackHandler): def on_chat_model_start(self, serialized, messages, **kwargs): for message in messages[0]: color_print(message.pretty_repr(), color="green") class NodeLogger(BaseCallbackHandler): def on_llm_end(self, response, **kwargs): # response of type LLMResult for generations in response.generations: # response.generations of type List[List[Generations]] becuase "each input could have multiple candidate generations" for generation in generations: generated_text = generation.message.content # token_usage = generation.message.response_metadata["token_usage"] color_print(generated_text, color="yellow") def on_agent_action(self, action, **kwargs): color_print(action.log, color="pink") def on_agent_finish(self, finish, **kwargs): color_print(finish, color="red") def on_tool_start(self, serialized, input_str, **kwargs): tool_name = serialized.get("name") color_print(f"Calling {tool_name} with inputs: {input_str}", color="pink") def on_tool_end(self, output, **kwargs): output = str(output) color_print(output, color="blue") def check_or_create_path(path=None): # Set a default path if none is provided if path is None: path = os.path.join(os.getcwd(), "tmp_directory") # Check if the path exists if not os.path.exists(path): # If it doesn't exist, create the directory os.makedirs(path) print(f"Directory created at: {path}") else: print(f"Directory already exists at: {path}") return path def langchain_to_gradio_message(message): # Build the title and content based on the message type if isinstance(message.content, list): # For a message with multiple content items (like text and tool use) gradio_messages = [] for item in message.content: gradio_message = { "role": "user" if message.type == "human" else "assistant", "content": "", "metadata": {}, } if item["type"] == "text": item["text"] = item["text"].replace("", "\n") item["text"] = item["text"].replace("", "\n") gradio_message["content"] += f"{item['text']}\n" gradio_messages.append(gradio_message) elif item["type"] == "tool_use": if item["name"] == "run_python_repl": gradio_message["metadata"]["title"] = "🛠️ Writing code..." # input = "```python {code_block}```\n".format(code_block=item['input']["command"]) gradio_message["metadata"]["log"] = "Executing Code block..." gradio_message["content"] = f"##### Code: \n ```python \n {item['input']['command']} \n``` \n" else: gradio_message["metadata"]["title"] = f"🛠️ Used tool ```{item['name']}```" to_print = ";".join([i + ": " + str(j) for i, j in item["input"].items()]) gradio_message["metadata"]["log"] = f"🔍 Input -- {to_print}\n" gradio_message["metadata"]["status"] = "pending" gradio_messages.append(gradio_message) else: gradio_message = { "role": "user" if message.type == "human" else "assistant", "content": "", "metadata": {}, } print(message) content = message.content content = content.replace("", "\n") content = content.replace("", "\n") content = content.replace("", "\n") content = content.replace("", "\n") gradio_message["content"] = content gradio_messages = [gradio_message] return gradio_messages def parse_hpo_obo(file_path): """Parse the HPO OBO file and create a dictionary mapping HP IDs to phenotype descriptions. Args: file_path (str): Path to the HPO OBO file. Returns: dict: A dictionary where keys are HP IDs and values are phenotype descriptions. """ hp_dict = {} current_id = None current_name = None with open(file_path) as file: for line in file: line = line.strip() if line.startswith("[Term]"): # If a new term block starts, save the previous term if current_id and current_name: hp_dict[current_id] = current_name current_id = None current_name = None elif line.startswith("id: HP:"): current_id = line.split(": ")[1] elif line.startswith("name:"): current_name = line.split(": ", 1)[1] # Add the last term to the dictionary if current_id and current_name: hp_dict[current_id] = current_name return hp_dict def textify_api_dict(api_dict): """Convert a nested API dictionary to a nicely formatted string.""" lines = [] for category, methods in api_dict.items(): lines.append(f"Import file: {category}") lines.append("=" * (len("Import file: ") + len(category))) for method in methods: lines.append(f"Method: {method.get('name', 'N/A')}") lines.append(f" Description: {method.get('description', 'No description provided.')}") # Process required parameters req_params = method.get("required_parameters", []) if req_params: lines.append(" Required Parameters:") for param in req_params: param_name = param.get("name", "N/A") param_type = param.get("type", "N/A") param_desc = param.get("description", "No description") param_default = param.get("default", "None") lines.append(f" - {param_name} ({param_type}): {param_desc} [Default: {param_default}]") # Process optional parameters opt_params = method.get("optional_parameters", []) if opt_params: lines.append(" Optional Parameters:") for param in opt_params: param_name = param.get("name", "N/A") param_type = param.get("type", "N/A") param_desc = param.get("description", "No description") param_default = param.get("default", "None") lines.append(f" - {param_name} ({param_type}): {param_desc} [Default: {param_default}]") lines.append("") # Empty line between methods lines.append("") # Extra empty line after each category return "\n".join(lines) def read_module2api(): fields = [ "support_tools", "pathology" ] module2api = {} for field in fields: module_name = f"histopath.tool.tool_description.{field}" module = importlib.import_module(module_name) module2api[f"histopath.tool.{field}"] = module.description return module2api