change gpu memory display format

gpu_info.py

@@ -1,19 +1,19 @@
 from subprocess import check_output
 from threading import Timer
-from typing import Callable, List
+from typing import Callable, List, Tuple
 
 
 def get_gpu_memory() -> List[int]:
     """
-    Get the free GPU memory (VRAM) in MiB
+    Get the used and total GPU memory (VRAM) in MiB
 
-    :return memory_free_values: List of free GPU memory (VRAM) in MiB
+    :return memory_values: List of used and total GPU memory (VRAM) in MiB
     """
 
-    command = "nvidia-smi --query-gpu=memory.free --format=csv,noheader,nounits"
-    memory_free_info = check_output(command.split()).decode("ascii").replace("\r", "").split("\n")[:-1]
-    memory_free_values = list(map(int, memory_free_info))
-    return memory_free_values
+    command = "nvidia-smi --query-gpu=memory.used,memory.total --format=csv,noheader,nounits"
+    memory_info = check_output(command.split()).decode("ascii").replace("\r", "").split("\n")[:-1]
+    memory_values = list(map(lambda x: tuple(map(int, x.split(","))), memory_info))
+    return memory_values
 
 
 class RepeatingTimer(Timer):
@@ -27,7 +27,7 @@ class RepeatingTimer(Timer):
 gpu_memory_watcher: RepeatingTimer = None
 
 
-def watch_gpu_memory(interval: int = 1, callback: Callable[[List[int]], None] = None) -> RepeatingTimer:
+def watch_gpu_memory(interval: int = 1, callback: Callable[[List[Tuple[int, int]]], None] = None) -> RepeatingTimer:
     """
     Start a repeating timer to watch the GPU memory usage
 

helper.py

@@ -417,7 +417,7 @@ def generate_video(
     )
 
     # watch gpu memory
-    watcher = watch_gpu_memory(10, lambda x: log.debug(f"GPU memory usage: {x} (MiB)"))
+    watcher = watch_gpu_memory(10, lambda x: log.debug(f"GPU memory (used, total): {x} (MiB)"))
 
     # start inference
     if chunking <= 0: