feat: Add chunk-size parameters and improve file type filtering

- Add --chunk-size and --chunk-overlap parameters to all RAG examples
- Preserve original default values for each data source:
  - Document: 256/128 (optimized for general documents)
  - Email: 256/25 (smaller overlap for email threads)
  - Browser: 256/128 (standard for web content)
  - WeChat: 192/64 (smaller chunks for chat messages)
- Make --file-types optional filter instead of restriction in document_rag
- Update README to clarify interactive mode and parameter usage
- Fix LLM default model documentation (gpt-4o, not gpt-4o-mini)
This commit is contained in:
Andy Lee
2025-07-29 18:31:56 -07:00
parent 8c152c7a31
commit 274bbb19ea
5 changed files with 63 additions and 32 deletions

View File

@@ -42,6 +42,12 @@ class WeChatRAG(BaseRAGExample):
action="store_true",
help="Force re-export of WeChat data even if exports exist",
)
wechat_group.add_argument(
"--chunk-size", type=int, default=192, help="Text chunk size (default: 192)"
)
wechat_group.add_argument(
"--chunk-overlap", type=int, default=64, help="Text chunk overlap (default: 64)"
)
def _export_wechat_data(self, export_dir: Path) -> bool:
"""Export WeChat data using wechattweak-cli."""
@@ -120,7 +126,9 @@ class WeChatRAG(BaseRAGExample):
print(f"Loaded {len(documents)} chat entries")
# Convert to text chunks
all_texts = create_text_chunks(documents)
all_texts = create_text_chunks(
documents, chunk_size=args.chunk_size, chunk_overlap=args.chunk_overlap
)
return all_texts