fixing chunking token issues within limit for embedding models

packages/leann-core/src/leann/embedding_compute.py

@@ -96,7 +96,6 @@ def get_model_token_limit(model_name: str) -> int:
     logger.warning(f"Unknown model '{model_name}', using default 512 token limit")
     return 512
 
-
 # Set up logger with proper level
 logger = logging.getLogger(__name__)
 LOG_LEVEL = os.getenv("LEANN_LOG_LEVEL", "WARNING").upper()
@@ -867,9 +866,7 @@ def compute_embeddings_ollama(
                 if retry_count >= max_retries:
                     # Enhanced error detection for token limit violations
                     error_msg = str(e).lower()
-                    if "token" in error_msg and (
+                    if "token" in error_msg and ("limit" in error_msg or "exceed" in error_msg or "length" in error_msg):
-                        "limit" in error_msg or "exceed" in error_msg or "length" in error_msg
-                    ):
                         logger.error(
                             f"Token limit exceeded for batch. Error: {e}. "
                             f"Consider reducing chunk sizes or check token truncation."