Update KB service to use new ES indices

ask_candid/services/knowledge_base.py

@@ -6,7 +6,7 @@ import logging
 from langchain_core.documents import Document
 
 from ask_candid.base.retrieval.elastic import (
-    build_sparse_vector_query,
+    # build_sparse_vector_query,
     build_sparse_vector_and_text_query,
     news_query_builder,
     issuelab_query_builder,
@@ -15,7 +15,7 @@ from ask_candid.base.retrieval.elastic import (
 from ask_candid.base.retrieval.sparse_lexical import SpladeEncoder
 from ask_candid.base.retrieval.schemas import ElasticHitsResult
 import ask_candid.base.retrieval.sources as S
-from ask_candid.base.config.connections import SEMANTIC_ELASTIC_QA, NEWS_ELASTIC
+from ask_candid.base.config.connections import SEMANTIC_ELASTIC, ELSER_INFERENCE_ID, NEWS_ELASTIC
 from ask_candid.services.small_lm import CandidSmallLanguageModel
 
 SourceNames = Literal[
@@ -32,7 +32,6 @@ logger = logging.getLogger(__name__)
 logger.setLevel(logging.INFO)
 
 
-# TODO remove
 def get_context(field_name: str, hit: ElasticHitsResult, context_length: int = 1024, add_context: bool = True) -> str:
     """Pads the relevant chunk of text with context before and after
 
@@ -123,18 +122,31 @@ def generate_queries(
                 semantic_fields=S.CandidBlogConfig.semantic_fields,
                 text_fields=S.CandidBlogConfig.text_fields,
                 highlight_fields=S.CandidBlogConfig.highlight_fields,
-                excluded_fields=S.CandidBlogConfig.excluded_fields
+                excluded_fields=S.CandidBlogConfig.excluded_fields,
+                inference_id=ELSER_INFERENCE_ID
             )
             q["size"] = 5
             vector_queries.extend([{"index": S.CandidBlogConfig.index_name}, q])
         elif source_name == "Candid Help":
-            q = build_sparse_vector_query(query=query, fields=S.CandidHelpConfig.semantic_fields)
-            q["_source"] = {"excludes": ["embeddings"]}
+            q = build_sparse_vector_and_text_query(
+                query=query,
+                semantic_fields=S.CandidHelpConfig.semantic_fields,
+                text_fields=S.CandidHelpConfig.text_fields,
+                highlight_fields=S.CandidHelpConfig.highlight_fields,
+                excluded_fields=S.CandidHelpConfig.excluded_fields,
+                inference_id=ELSER_INFERENCE_ID
+            )
             q["size"] = 5
             vector_queries.extend([{"index": S.CandidHelpConfig.index_name}, q])
         elif source_name == "Candid Learning":
-            q = build_sparse_vector_query(query=query, fields=S.CandidLearningConfig.semantic_fields)
-            q["_source"] = {"excludes": ["embeddings"]}
+            q = build_sparse_vector_and_text_query(
+                query=query,
+                semantic_fields=S.CandidLearningConfig.semantic_fields,
+                text_fields=S.CandidLearningConfig.text_fields,
+                highlight_fields=S.CandidLearningConfig.highlight_fields,
+                excluded_fields=S.CandidLearningConfig.excluded_fields,
+                inference_id=ELSER_INFERENCE_ID
+            )
             q["size"] = 5
             vector_queries.extend([{"index": S.CandidLearningConfig.index_name}, q])
         elif source_name == "Candid News":
@@ -166,7 +178,8 @@ def generate_queries(
                 semantic_fields=S.YoutubeConfig.semantic_fields,
                 text_fields=S.YoutubeConfig.text_fields,
                 highlight_fields=S.YoutubeConfig.highlight_fields,
-                excluded_fields=S.YoutubeConfig.excluded_fields
+                excluded_fields=S.YoutubeConfig.excluded_fields,
+                inference_id=ELSER_INFERENCE_ID
             )
             q["size"] = 5
             vector_queries.extend([{"index": S.YoutubeConfig.index_name}, q])
@@ -215,7 +228,7 @@ def run_search(
 
     results = []
     if vector_searches is not None and len(vector_searches) > 0:
-        hits = multi_search_base(queries=vector_searches, credentials=SEMANTIC_ELASTIC_QA)
+        hits = multi_search_base(queries=vector_searches, credentials=SEMANTIC_ELASTIC)
         for hit in _msearch_response_generator(responses=hits):
             results.append(hit)
     if non_vector_searches is not None and len(non_vector_searches) > 0:
@@ -368,6 +381,7 @@ def process_hit(hit: ElasticHitsResult) -> Document:
         )
     elif "blog" in hit.index:
         highlight = hit.highlight or {}
+        blog_url = hit.source.get("link", "")
         doc = Document(
             page_content='\n\n'.join([
                 hit.source.get("title_summary_tags_text", ""),
@@ -378,35 +392,35 @@ def process_hit(hit: ElasticHitsResult) -> Document:
                 "title": hit.source.get("title", ""),
                 "source": "Candid Blog",
                 "source_id": hit.source["id"],
-                "url": hit.source["link"]
+                "url": blog_url
             }
         )
-    elif "candid-learning" in hit.index:
+    elif "learning" in hit.index:
+        highlight = hit.highlight or {}
         doc = Document(
             page_content='\n\n'.join([
-                hit.source.get("title", ""),
-                hit.source.get("staff_recommendations", ""),
-                hit.source.get("training_topics", ""),
-                get_context("content", hit, context_length=12)
+                hit.source.get("semantic_title_short_description", ""),
+                ' '.join(highlight.get("semantic_lessons_content", []))
             ]),
             metadata={
                 "title": hit.source["title"],
                 "source": "Candid Learning",
-                "source_id": hit.source["post_id"],
-                "url": hit.source.get("url", "")
+                "source_id": hit.source["course_id"],
+                "url": hit.source.get("course_url", "")
             }
         )
-    elif "candid-help" in hit.index:
+    elif "help" in hit.index:
+        highlight = hit.highlight or {}
         doc = Document(
             page_content='\n\n'.join([
-                hit.source.get("combined_article_description", ""),
-                get_context("content", hit, context_length=12)
+                hit.source.get("semantic_title_summary_question_category", ""),
+                ' '.join(highlight.get("semantic_content", []))
             ]),
             metadata={
                 "title": hit.source.get("title", ""),
                 "source": "Candid Help",
-                "source_id": hit.source["id"],
-                "url": hit.source.get("link", "")
+                "source_id": hit.source["article_id"],
+                "url": f"""https://help.candid.org/s/article/{hit.source.get("url", "")}"""
             }
         )
     elif "news" in hit.index: