Add indexing

app.py

@@ -1,12 +1,9 @@
+import glob
 import logging
 import sys
 
 import streamlit as st
-from haystack import Document
 from haystack import Pipeline
-from haystack.document_stores import InMemoryDocumentStore
-from haystack.nodes import EmbeddingRetriever
-from haystack.nodes import FARMReader
 
 logging.basicConfig(
     level=logging.DEBUG,
@@ -17,29 +14,26 @@ logging.basicConfig(
 
 p = None
 
+
 def app_init():
-    docs = [Document(id='1', content='His name is John.'),
-            Document(id='2', content='Her name is Jane.'),
-            Document(id='3', content='My name is Haystack.')]
-    ds = InMemoryDocumentStore()
-    ds.write_documents(docs)
-    retriever = EmbeddingRetriever(
-        document_store=ds,
-        embedding_model="sentence-transformers/multi-qa-mpnet-base-dot-v1",
-        model_format="sentence_transformers",
-    )
-    ds.update_embeddings(retriever)
-    reader = FARMReader("deepset/minilm-uncased-squad2", use_gpu=False)
-    p = Pipeline()
-    p.add_node(component=retriever, name='retriever', inputs=['Query'])
-    p.add_node(component=reader, name='reader', inputs=['retriever'])
+    indexing_pipeline = Pipeline.load_from_yaml("pipeline.yaml", pipeline_name="indexing")
+    file_paths = glob.glob("data/*")
+    ds = indexing_pipeline.get_node("DocumentStore")
+    ds.delete_all_documents()
+    indexing_pipeline.run(file_paths=file_paths)
+    ds.update_embeddings(indexing_pipeline.get_node("Retriever"))
+    ds.save(config_path="my_faiss_config.json", index_path="my_faiss_index.faiss")
+
+    global p
+    p = Pipeline.load_from_yaml("pipeline.yaml", pipeline_name="query")
 
 
 def main():
     app_init()
     st.title("Haystack Demo")
     input = st.text_input("Query ...")
-    st.text(p.run(input))
+    st.text(p.run(str(input or "test")))
+
 
 if __name__ == "__main__":
-    main()
+    main()

my_faiss_config.json

@@ -0,0 +1 @@
+{"faiss_config_path": "my_faiss_config.json", "embedding_dim": 768}

requirements.txt

@@ -1 +1 @@
-farm-haystack
+farm-haystack[faiss]