#!/usr/bin/env python3 """ CI pytest wrapper with comprehensive hang detection and cleanup. Designed to prevent CI hangs due to subprocess or cleanup issues. """ import os import signal import subprocess import sys import time def cleanup_all_processes(): """Aggressively cleanup all related processes.""" print("๐Ÿงน [CLEANUP] Performing aggressive cleanup...") # Kill by pattern - use separate calls to avoid shell injection patterns = [ "embedding_server", "hnsw_embedding", "zmq", "python.*pytest", "scripts/ci_debug_pytest", ] for pattern in patterns: try: subprocess.run(["pkill", "-9", "-f", pattern], timeout=5, capture_output=True) except Exception: pass # Clean up any hanging Python processes with specific patterns try: result = subprocess.run(["ps", "aux"], capture_output=True, text=True, timeout=5) lines = result.stdout.split("\n") for line in lines: if "python" in line and ("test_" in line or "pytest" in line or "embedding" in line): try: pid = line.split()[1] subprocess.run(["kill", "-9", pid], timeout=2) except Exception: pass except Exception: pass print("๐Ÿงน [CLEANUP] Cleanup completed") def run_pytest_with_monitoring(pytest_args): """Run pytest with comprehensive monitoring and timeout handling.""" # Pre-test cleanup print("๐Ÿงน [WRAPPER] Pre-test cleanup...") cleanup_all_processes() time.sleep(2) # Show pre-test state print("๐Ÿ“Š [WRAPPER] Pre-test process state:") try: result = subprocess.run(["ps", "aux"], capture_output=True, text=True, timeout=5) relevant_lines = [ line for line in result.stdout.split("\n") if "python" in line or "embedding" in line or "zmq" in line ] if relevant_lines: for line in relevant_lines[:5]: # Show first 5 matches print(f" {line}") else: print(" No relevant processes found") except Exception: print(" Process check failed") # Setup signal handlers for cleanup def signal_handler(signum, frame): print(f"\n๐Ÿ’ฅ [WRAPPER] Received signal {signum}, cleaning up...") cleanup_all_processes() sys.exit(128 + signum) signal.signal(signal.SIGTERM, signal_handler) signal.signal(signal.SIGINT, signal_handler) # Run pytest with monitoring print(f"๐Ÿš€ [WRAPPER] Starting pytest with args: {pytest_args}") try: # Use Popen for better control cmd = [sys.executable, "scripts/ci_debug_pytest.py", *pytest_args] process = subprocess.Popen( cmd, stdout=sys.stdout, stderr=sys.stderr, preexec_fn=os.setsid if hasattr(os, "setsid") else None, ) # Monitor with timeout start_time = time.time() timeout = 600 # 10 minutes monitor_interval = 10 # Check every 10 seconds while True: # Check if process completed return_code = process.poll() if return_code is not None: print(f"โœ… [WRAPPER] Pytest completed with return code: {return_code}") break # Check for timeout elapsed = time.time() - start_time if elapsed > timeout: print(f"๐Ÿ’ฅ [WRAPPER] Pytest timed out after {elapsed:.1f}s") # Try graceful termination try: print("๐Ÿ”„ [WRAPPER] Attempting graceful termination...") process.terminate() try: process.wait(timeout=10) except subprocess.TimeoutExpired: print("๐Ÿ’€ [WRAPPER] Graceful termination failed, force killing...") process.kill() process.wait() except Exception as e: print(f"โš ๏ธ [WRAPPER] Error during termination: {e}") return_code = 124 # timeout exit code break # Monitor progress if int(elapsed) % 30 == 0: # Every 30 seconds print(f"๐Ÿ“Š [WRAPPER] Monitor check: {elapsed:.0f}s elapsed, pytest still running") time.sleep(monitor_interval) # Post-test cleanup verification print("๐Ÿ” [WRAPPER] Post-test cleanup verification...") time.sleep(2) try: result = subprocess.run(["ps", "aux"], capture_output=True, text=True, timeout=5) remaining = [ line for line in result.stdout.split("\n") if "python" in line and ("pytest" in line or "embedding" in line) ] if remaining: print(f"โš ๏ธ [WRAPPER] Found {len(remaining)} remaining processes:") for line in remaining[:3]: # Show first 3 print(f" {line}") print("๐Ÿ’€ [WRAPPER] Performing final cleanup...") cleanup_all_processes() else: print("โœ… [WRAPPER] No remaining processes found") except Exception: print("โš ๏ธ [WRAPPER] Post-test verification failed, performing cleanup anyway") cleanup_all_processes() return return_code except Exception as e: print(f"๐Ÿ’ฅ [WRAPPER] Error running pytest: {e}") cleanup_all_processes() return 1 def main(): """Main entry point.""" if len(sys.argv) < 2: print("Usage: ci_pytest_wrapper.py ") return 1 pytest_args = sys.argv[1:] print(f"๐ŸŽฏ [WRAPPER] CI pytest wrapper starting with args: {pytest_args}") return run_pytest_with_monitoring(pytest_args) if __name__ == "__main__": sys.exit(main())