fix: improve signal handling with process groups for hanzo net

- Use process groups (setsid) to ensure all child processes receive signals
- Send SIGTERM/SIGKILL to entire process group on shutdown
- Handles nested subprocess properly (hanzo -> hanzo/net)
- Test confirms graceful shutdown now works with Ctrl-C

Fully fixes the issue where Ctrl-C wouldn't stop hanzo net properly
This commit is contained in:
2025-08-14 16:03:42 -05:00
parent 72fe7a2656
commit f141dde84c
2 changed files with 59 additions and 8 deletions
+18 -4
View File
@@ -296,19 +296,33 @@ async def start_compute_node(
cmd_args.extend(["--default-model", models[0]])
# Run net command with detected python in a more signal-friendly way
process = subprocess.Popen(cmd_args, env=env)
# Create new process group for better signal handling
process = subprocess.Popen(
cmd_args,
env=env,
preexec_fn=os.setsid if hasattr(os, 'setsid') else None
)
# Set up signal handlers to forward to subprocess
# Set up signal handlers to forward to subprocess group
def signal_handler(signum, frame):
if process.poll() is None: # Process is still running
console.print("\n[yellow]Stopping hanzo net...[/yellow]")
process.terminate() # Try graceful termination first
try:
# Send signal to entire process group
if hasattr(os, 'killpg'):
os.killpg(os.getpgid(process.pid), signal.SIGTERM)
else:
process.terminate()
process.wait(timeout=5) # Wait up to 5 seconds
except subprocess.TimeoutExpired:
console.print("[yellow]Force stopping...[/yellow]")
process.kill() # Force kill if needed
if hasattr(os, 'killpg'):
os.killpg(os.getpgid(process.pid), signal.SIGKILL)
else:
process.kill()
process.wait()
except ProcessLookupError:
pass # Process already terminated
raise KeyboardInterrupt
# Register signal handlers
+41 -4
View File
@@ -11,12 +11,19 @@ def test_signal_handling():
print("Testing signal handling for 'hanzo net'...")
print("Starting hanzo net process...")
# Start hanzo net
# Start hanzo net using the local module
import os
env = os.environ.copy()
env['PYTHONPATH'] = os.path.join(os.path.dirname(__file__), 'pkg/hanzo/src') + ':' + env.get('PYTHONPATH', '')
process = subprocess.Popen(
[sys.executable, "-m", "hanzo", "net"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True
text=True,
env=env,
cwd=os.path.dirname(__file__),
preexec_fn=os.setsid # Create new process group
)
# Give it time to start
@@ -29,8 +36,38 @@ def test_signal_handling():
print(f"Stderr: {stderr}")
return False
print("Sending SIGINT (Ctrl-C) to process...")
process.send_signal(signal.SIGINT)
print("Process is running, checking output...")
# Read any initial output without blocking
import select
import fcntl
import os as os2
# Make stdout non-blocking
fl = fcntl.fcntl(process.stdout.fileno(), fcntl.F_GETFL)
fcntl.fcntl(process.stdout.fileno(), fcntl.F_SETFL, fl | os2.O_NONBLOCK)
# Try to read initial output
try:
initial_output = process.stdout.read()
if initial_output:
print(f"Initial stdout: {initial_output[:500]}")
except:
pass
# Make stderr non-blocking too
fl = fcntl.fcntl(process.stderr.fileno(), fcntl.F_GETFL)
fcntl.fcntl(process.stderr.fileno(), fcntl.F_SETFL, fl | os2.O_NONBLOCK)
try:
initial_stderr = process.stderr.read()
if initial_stderr:
print(f"Initial stderr: {initial_stderr[:500]}")
except:
pass
print("Sending SIGINT (Ctrl-C) to process group...")
# Send signal to the entire process group
os.killpg(os.getpgid(process.pid), signal.SIGINT)
# Wait for graceful shutdown (up to 10 seconds)
try: