Datastore reproduce (#3)

* fix: diskann zmq port and passages

* feat: auto discovery of packages and fix passage gen for diskann

* docs: embedding pruning

* refactor: passage structure

* feat: reproducible research datas, rpj_wiki & dpr

* refactor: chat and base searcher

* feat: chat on mps

test/check_embedding_dimension.py

@@ -1,24 +0,0 @@
-from llama_index.core import VectorStoreIndex, Document
-from llama_index.core.embeddings import resolve_embed_model
-
-# Check the default embedding model
-embed_model = resolve_embed_model("default")
-print(f"Default embedding model: {embed_model}")
-
-# Create a simple test document
-doc = Document(text="This is a test document")
-
-# Get embedding dimension
-try:
-    # Test embedding
-    test_embedding = embed_model.get_text_embedding("test")
-    print(f"Embedding dimension: {len(test_embedding)}")
-    print(f"Embedding type: {type(test_embedding)}")
-except Exception as e:
-    print(f"Error getting embedding: {e}")
-
-# Alternative way to check dimension
-if hasattr(embed_model, 'embed_dim'):
-    print(f"Model embed_dim attribute: {embed_model.embed_dim}")
-elif hasattr(embed_model, 'dimension'):
-    print(f"Model dimension attribute: {embed_model.dimension}") 

test/check_embedding_model.py

@@ -1,20 +0,0 @@
-from llama_index.core import VectorStoreIndex, Document
-from llama_index.core.embeddings import resolve_embed_model
-
-# Check the default embedding model
-embed_model = resolve_embed_model("default")
-print(f"Default embedding model: {embed_model}")
-
-# Create a simple test
-doc = Document(text="This is a test document")
-index = VectorStoreIndex.from_documents([doc])
-
-# Get the embedding model from the index
-index_embed_model = index.embed_model
-print(f"Index embedding model: {index_embed_model}")
-
-# Check if it's OpenAI or local
-if hasattr(index_embed_model, 'model_name'):
-    print(f"Model name: {index_embed_model.model_name}")
-else:
-    print(f"Embedding model type: {type(index_embed_model)}")