rm useless

This commit is contained in:
yichuan520030910320
2025-07-06 06:47:20 +00:00
parent af0959818d
commit 95a653993a
2 changed files with 2 additions and 1 deletions

View File

@@ -23,7 +23,7 @@ file_extractor: dict[str, BaseReader] = {
".xlsx": reader,
}
node_parser = DoclingNodeParser(
chunker=HybridChunker(tokenizer="Qwen/Qwen3-Embedding-4B", max_tokens=64)
chunker=HybridChunker(tokenizer="Qwen/Qwen3-Embedding-4B", max_tokens=128)
)
print("Loading documents...")
documents = SimpleDirectoryReader(

View File

@@ -32,6 +32,7 @@ def _compute_embeddings(chunks: List[str], model_name: str) -> np.ndarray:
else:
from sentence_transformers import SentenceTransformer
model = SentenceTransformer(model_name)
model = model.half()
print(f"INFO: Computing embeddings for {len(chunks)} chunks using SentenceTransformer model '{model_name}'...")
embeddings = model.encode(chunks, show_progress_bar=True)