recover mess

modular_graph_and_candidates.py

@@ -45,7 +45,7 @@ import spaces
 # ────────────────────────────────────────────────────────────────────────────────
 # CONFIG
 # ───────────────────────────────────────────────────────────────────────────────
-SIM_DEFAULT     = 0.78   # Jaccard similarity threshold
+SIM_DEFAULT     = 0.5   # similarity threshold
 PIXEL_MIN_HITS  = 0      # multimodal trigger ("pixel_values")
 HTML_DEFAULT = "d3_modular_graph.html"
 
@@ -126,26 +126,24 @@ def embedding_similarity_clusters(models_root: Path, missing: List[str], thr: fl
         texts[name] = code.strip() or " "
 
     names = list(texts)
-    names = names[:90]  # Limit before processing
     all_embeddings = []
 
     print(f"Encoding embeddings for {len(names)} models...")
     batch_size = 8
     
-    for i in tqdm(range(0, len(names), batch_size), desc="Models", leave=False):
-        model_name = names[i]
-        text_len = len(texts[model_name])
+    for i in tqdm(range(0, len(names), batch_size), desc="Batches", leave=False):
+        batch_names = names[i:i+batch_size]
+        batch_texts = [texts[name] for name in batch_names]
         
         try:
-            print(f"Processing {model_name} (text length: {text_len})")
-            batch = [texts[model_name]]
-            emb = model.encode(batch, convert_to_numpy=True, show_progress_bar=False)
+            print(f"Processing batch: {batch_names}")
+            emb = model.encode(batch_texts, convert_to_numpy=True, show_progress_bar=False)
             all_embeddings.append(emb)
-            print(f"✓ Completed {model_name}")
+            print(f"✓ Completed batch of {len(batch_names)} models")
         except Exception as e:
-            print(f"⚠️  GPU worker error for {model_name}: {type(e).__name__}: {e}")
-            # Create zero embedding as placeholder to maintain consistency
-            zero_emb = np.zeros((1, model.get_sentence_embedding_dimension()), dtype=np.float32)
+            print(f"⚠️  GPU worker error for batch {batch_names}: {type(e).__name__}: {e}")
+            # Create zero embeddings for all models in failed batch
+            zero_emb = np.zeros((len(batch_names), model.get_sentence_embedding_dimension()), dtype=np.float32)
             all_embeddings.append(zero_emb)
 
     embeddings = np.vstack(all_embeddings).astype(np.float32)
@@ -376,7 +374,7 @@ function updateVisibility() {
 }
 document.getElementById('toggleRed').addEventListener('change', updateVisibility);
 
-const HF_LOGO_URI = "hf-logo.png";
+const HF_LOGO_URI = "static/hf-logo.svg";
 const graph = __GRAPH_DATA__;
 const W = innerWidth, H = innerHeight;
 const svg = d3.select('#dependency').call(d3.zoom().on('zoom', e => g.attr('transform', e.transform)));