mirror of
https://github.com/hanzoai/python-sdk.git
synced 2026-10-23 20:23:00 +00:00
fix: improve signal handling for hanzo net/node commands
- Add proper SIGINT/SIGTERM signal handlers - Use subprocess.Popen instead of subprocess.run for better control - Implement graceful shutdown with 5-second timeout before force kill - Handle signals in both async and subprocess modes - Add test script for signal handling verification Fixes issue where Ctrl-C wouldn't properly stop hanzo net
This commit is contained in:
@@ -2,6 +2,9 @@
|
||||
|
||||
import sys
|
||||
import asyncio
|
||||
import signal
|
||||
import os
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
import click
|
||||
@@ -115,7 +118,11 @@ def serve(ctx, name: str, port: int):
|
||||
@click.pass_context
|
||||
def net(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
|
||||
"""Start the Hanzo Network distributed AI compute node."""
|
||||
asyncio.run(start_compute_node(ctx, name, port, network, models, max_jobs))
|
||||
try:
|
||||
asyncio.run(start_compute_node(ctx, name, port, network, models, max_jobs))
|
||||
except KeyboardInterrupt:
|
||||
# Already handled in start_compute_node
|
||||
pass
|
||||
|
||||
|
||||
@cli.command()
|
||||
@@ -133,7 +140,11 @@ def net(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
|
||||
@click.pass_context
|
||||
def node(ctx, name: str, port: int, network: str, models: tuple, max_jobs: int):
|
||||
"""Alias for 'hanzo net' - Start as a compute node for the Hanzo network."""
|
||||
asyncio.run(start_compute_node(ctx, name, port, network, models, max_jobs))
|
||||
try:
|
||||
asyncio.run(start_compute_node(ctx, name, port, network, models, max_jobs))
|
||||
except KeyboardInterrupt:
|
||||
# Already handled in start_compute_node
|
||||
pass
|
||||
|
||||
|
||||
async def start_compute_node(
|
||||
@@ -204,8 +215,40 @@ async def start_compute_node(
|
||||
console.print("API: http://localhost:52415/v1/chat/completions")
|
||||
console.print("\nPress Ctrl+C to stop\n")
|
||||
|
||||
# Run net
|
||||
await net_run()
|
||||
# Set up signal handlers for async version
|
||||
stop_event = asyncio.Event()
|
||||
|
||||
def async_signal_handler(signum, frame):
|
||||
console.print("\n[yellow]Stopping hanzo net...[/yellow]")
|
||||
stop_event.set()
|
||||
|
||||
signal.signal(signal.SIGINT, async_signal_handler)
|
||||
signal.signal(signal.SIGTERM, async_signal_handler)
|
||||
|
||||
# Run net with proper signal handling
|
||||
try:
|
||||
net_task = asyncio.create_task(net_run())
|
||||
stop_task = asyncio.create_task(stop_event.wait())
|
||||
|
||||
# Wait for either net to complete or stop signal
|
||||
done, pending = await asyncio.wait(
|
||||
[net_task, stop_task],
|
||||
return_when=asyncio.FIRST_COMPLETED
|
||||
)
|
||||
|
||||
# Cancel pending tasks
|
||||
for task in pending:
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
|
||||
# Check if we stopped due to signal
|
||||
if stop_task in done:
|
||||
console.print("[green]✓[/green] Node stopped gracefully")
|
||||
except asyncio.CancelledError:
|
||||
console.print("[yellow]Cancelled[/yellow]")
|
||||
finally:
|
||||
sys.argv = original_argv
|
||||
else:
|
||||
@@ -252,12 +295,32 @@ async def start_compute_node(
|
||||
if models:
|
||||
cmd_args.extend(["--default-model", models[0]])
|
||||
|
||||
# Run net command with detected python
|
||||
process = subprocess.run(cmd_args, env=env, check=False)
|
||||
|
||||
if process.returncode != 0 and process.returncode != -2: # -2 is Ctrl+C
|
||||
# Run net command with detected python in a more signal-friendly way
|
||||
process = subprocess.Popen(cmd_args, env=env)
|
||||
|
||||
# Set up signal handlers to forward to subprocess
|
||||
def signal_handler(signum, frame):
|
||||
if process.poll() is None: # Process is still running
|
||||
console.print("\n[yellow]Stopping hanzo net...[/yellow]")
|
||||
process.terminate() # Try graceful termination first
|
||||
try:
|
||||
process.wait(timeout=5) # Wait up to 5 seconds
|
||||
except subprocess.TimeoutExpired:
|
||||
console.print("[yellow]Force stopping...[/yellow]")
|
||||
process.kill() # Force kill if needed
|
||||
process.wait()
|
||||
raise KeyboardInterrupt
|
||||
|
||||
# Register signal handlers
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
|
||||
# Wait for process to complete
|
||||
returncode = process.wait()
|
||||
|
||||
if returncode != 0 and returncode != -2: # -2 is Ctrl+C
|
||||
console.print(
|
||||
f"[red]Net exited with code {process.returncode}[/red]"
|
||||
f"[red]Net exited with code {returncode}[/red]"
|
||||
)
|
||||
|
||||
finally:
|
||||
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Test signal handling for hanzo net command."""
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
import signal
|
||||
import sys
|
||||
|
||||
def test_signal_handling():
|
||||
"""Test that Ctrl-C properly stops hanzo net."""
|
||||
print("Testing signal handling for 'hanzo net'...")
|
||||
print("Starting hanzo net process...")
|
||||
|
||||
# Start hanzo net
|
||||
process = subprocess.Popen(
|
||||
[sys.executable, "-m", "hanzo", "net"],
|
||||
stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE,
|
||||
text=True
|
||||
)
|
||||
|
||||
# Give it time to start
|
||||
time.sleep(3)
|
||||
|
||||
if process.poll() is not None:
|
||||
print(f"❌ Process exited early with code: {process.returncode}")
|
||||
stdout, stderr = process.communicate()
|
||||
print(f"Stdout: {stdout}")
|
||||
print(f"Stderr: {stderr}")
|
||||
return False
|
||||
|
||||
print("Sending SIGINT (Ctrl-C) to process...")
|
||||
process.send_signal(signal.SIGINT)
|
||||
|
||||
# Wait for graceful shutdown (up to 10 seconds)
|
||||
try:
|
||||
returncode = process.wait(timeout=10)
|
||||
if returncode == 0 or returncode == -2:
|
||||
print("✅ Process shut down gracefully!")
|
||||
return True
|
||||
else:
|
||||
print(f"⚠️ Process exited with code: {returncode}")
|
||||
return False
|
||||
except subprocess.TimeoutExpired:
|
||||
print("❌ Process did not shut down within 10 seconds")
|
||||
print("Force killing...")
|
||||
process.kill()
|
||||
process.wait()
|
||||
return False
|
||||
|
||||
if __name__ == "__main__":
|
||||
success = test_signal_handling()
|
||||
sys.exit(0 if success else 1)
|
||||
Reference in New Issue
Block a user