mirror of
https://github.com/zenlm/gym.git
synced 2026-11-04 17:11:44 +00:00
375 lines
11 KiB
Python
375 lines
11 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Test script for production features in Gym training platform.
|
|
Demonstrates distributed training, monitoring, model registry,
|
|
hyperparameter tuning, and checkpoint management.
|
|
"""
|
|
|
|
import os
|
|
import sys
|
|
import torch
|
|
import torch.nn as nn
|
|
import torch.optim as optim
|
|
from pathlib import Path
|
|
|
|
# Add src to path
|
|
sys.path.insert(0, str(Path(__file__).parent / "src"))
|
|
|
|
from gym.distributed import DistributedConfig, DistributedCoordinator
|
|
from gym.monitoring import MetricsCollector, TrainingMonitor
|
|
from gym.registry import ModelRegistry, ModelStatus
|
|
from gym.tuning import (
|
|
SearchStrategy,
|
|
ParameterSpec,
|
|
HyperparameterSearch,
|
|
EarlyStoppingCallback,
|
|
)
|
|
from gym.utils.checkpoint_manager import CheckpointManager, AutoResumeTrainer
|
|
|
|
|
|
# Simple test model
|
|
class SimpleModel(nn.Module):
|
|
"""Simple feedforward model for testing."""
|
|
|
|
def __init__(self, input_size=784, hidden_size=128, num_classes=10):
|
|
super().__init__()
|
|
self.fc1 = nn.Linear(input_size, hidden_size)
|
|
self.relu = nn.ReLU()
|
|
self.fc2 = nn.Linear(hidden_size, num_classes)
|
|
|
|
def forward(self, x):
|
|
x = self.fc1(x)
|
|
x = self.relu(x)
|
|
x = self.fc2(x)
|
|
return x
|
|
|
|
|
|
def test_distributed_training():
|
|
"""Test distributed training coordinator."""
|
|
print("\n=== Testing Distributed Training ===")
|
|
|
|
# Initialize coordinator
|
|
config = DistributedConfig(use_fsdp=False, mixed_precision=True)
|
|
coordinator = DistributedCoordinator(config)
|
|
|
|
# Initialize distributed training
|
|
if coordinator.initialize():
|
|
print(f"✓ Distributed coordinator initialized (rank={config.rank})")
|
|
|
|
# Create and wrap model
|
|
model = SimpleModel()
|
|
model = coordinator.wrap_model(model)
|
|
print("✓ Model wrapped for distributed training")
|
|
|
|
# Health check
|
|
health = coordinator.health_check()
|
|
print(f"✓ Health check: {health['cuda_available']=}, {health.get('communication_ok', True)=}")
|
|
|
|
# Cleanup
|
|
coordinator.cleanup()
|
|
print("✓ Distributed resources cleaned up")
|
|
|
|
|
|
def test_metrics_monitoring():
|
|
"""Test metrics collection and monitoring."""
|
|
print("\n=== Testing Metrics & Monitoring ===")
|
|
|
|
# Initialize metrics collector
|
|
metrics = MetricsCollector(
|
|
log_dir="./test_logs",
|
|
backends=["console", "json"],
|
|
log_interval=2,
|
|
)
|
|
|
|
# Create training monitor
|
|
monitor = TrainingMonitor(metrics)
|
|
|
|
# Simulate training loop
|
|
monitor.on_epoch_start(epoch=0)
|
|
|
|
for batch in range(5):
|
|
# Simulate batch training
|
|
loss = 1.0 - (batch * 0.1) # Decreasing loss
|
|
monitor.on_batch_end(loss=loss, batch_size=32, learning_rate=0.001)
|
|
|
|
# Simulate validation
|
|
val_metrics = {"loss": 0.5, "accuracy": 0.85}
|
|
is_best = monitor.on_validation(val_metrics)
|
|
print(f"✓ Validation metrics logged (is_best={is_best})")
|
|
|
|
# Log resources
|
|
metrics.log_resources()
|
|
print("✓ Resource metrics collected")
|
|
|
|
# Get summary
|
|
summary = metrics.get_summary()
|
|
print(f"✓ Metrics summary: {summary['total_steps']} steps logged")
|
|
|
|
metrics.close()
|
|
|
|
|
|
def test_model_registry():
|
|
"""Test model registry functionality."""
|
|
print("\n=== Testing Model Registry ===")
|
|
|
|
# Initialize registry
|
|
registry = ModelRegistry(base_path="./test_registry")
|
|
|
|
# Create and register model
|
|
model = SimpleModel()
|
|
model_id = registry.register(
|
|
model=model,
|
|
name="simple_model",
|
|
version="1.0.0",
|
|
architecture="feedforward",
|
|
metrics={"loss": 0.5, "accuracy": 0.85},
|
|
tags=["test", "production"],
|
|
description="Test model for production features",
|
|
author="gym_test",
|
|
)
|
|
print(f"✓ Model registered with ID: {model_id}")
|
|
|
|
# Update model status
|
|
registry.update_status(model_id, ModelStatus.STAGING)
|
|
print("✓ Model status updated to STAGING")
|
|
|
|
# Create alias
|
|
registry.create_alias("latest", model_id)
|
|
print("✓ Created alias 'latest' for model")
|
|
|
|
# Load model
|
|
state_dict, metadata = registry.load("latest")
|
|
print(f"✓ Model loaded: {metadata.name}:{metadata.version}")
|
|
|
|
# List models
|
|
models = registry.list_models()
|
|
print(f"✓ Found {len(models)} models in registry")
|
|
|
|
# Compare models (simulate second model)
|
|
model2 = SimpleModel(hidden_size=256)
|
|
model_id2 = registry.register(
|
|
model=model2,
|
|
name="simple_model",
|
|
version="2.0.0",
|
|
architecture="feedforward",
|
|
metrics={"loss": 0.4, "accuracy": 0.90},
|
|
parent_model_id=model_id,
|
|
)
|
|
|
|
comparison = registry.compare_models(model_id, model_id2)
|
|
print(f"✓ Model comparison: accuracy improved by {comparison['metric_diff']['accuracy']:.2f}")
|
|
|
|
|
|
def test_hyperparameter_search():
|
|
"""Test hyperparameter tuning."""
|
|
print("\n=== Testing Hyperparameter Search ===")
|
|
|
|
# Define parameter search space
|
|
param_specs = [
|
|
ParameterSpec("learning_rate", "log", (1e-4, 1e-2)),
|
|
ParameterSpec("batch_size", "categorical", [16, 32, 64, 128]),
|
|
ParameterSpec("hidden_size", "int", (64, 512)),
|
|
ParameterSpec("dropout", "float", (0.0, 0.5)),
|
|
]
|
|
|
|
# Initialize search
|
|
search = HyperparameterSearch(
|
|
param_specs=param_specs,
|
|
strategy=SearchStrategy.RANDOM,
|
|
metric_name="validation_loss",
|
|
mode="min",
|
|
n_trials=5,
|
|
log_dir="./test_hp_search",
|
|
)
|
|
|
|
# Run trials
|
|
early_stopping = EarlyStoppingCallback(patience=3)
|
|
|
|
for i in range(5):
|
|
# Get next trial
|
|
trial = search.suggest_next()
|
|
if trial is None:
|
|
break
|
|
|
|
print(f"✓ Trial {trial.trial_id}: {trial.parameters}")
|
|
|
|
# Start trial
|
|
search.start_trial(trial.trial_id)
|
|
|
|
# Simulate training with these hyperparameters
|
|
# In real scenario, you would train the model here
|
|
simulated_loss = 1.0 - (i * 0.15) # Improving loss
|
|
|
|
# Complete trial
|
|
search.complete_trial(
|
|
trial.trial_id,
|
|
metric=simulated_loss,
|
|
metadata={"epochs_trained": 10}
|
|
)
|
|
|
|
# Check early stopping
|
|
if early_stopping(simulated_loss):
|
|
print("✓ Early stopping triggered")
|
|
break
|
|
|
|
# Get best parameters
|
|
best_params = search.get_best_parameters()
|
|
print(f"✓ Best parameters found: {best_params}")
|
|
|
|
# Get summary
|
|
summary = search.get_summary()
|
|
print(f"✓ Search summary: {summary['completed_trials']} trials completed")
|
|
|
|
|
|
def test_checkpoint_manager():
|
|
"""Test checkpoint management and auto-resume."""
|
|
print("\n=== Testing Checkpoint Manager ===")
|
|
|
|
# Initialize checkpoint manager
|
|
ckpt_manager = CheckpointManager(
|
|
checkpoint_dir="./test_checkpoints",
|
|
max_checkpoints=3,
|
|
save_best=True,
|
|
save_latest=True,
|
|
)
|
|
|
|
# Create model and optimizer
|
|
model = SimpleModel()
|
|
optimizer = optim.Adam(model.parameters(), lr=0.001)
|
|
|
|
# Save checkpoints
|
|
for epoch in range(3):
|
|
step = epoch * 100
|
|
loss = 1.0 - (epoch * 0.2)
|
|
is_best = epoch == 2 # Last epoch is best
|
|
|
|
checkpoint_id = ckpt_manager.save(
|
|
model=model,
|
|
optimizer=optimizer,
|
|
epoch=epoch,
|
|
step=step,
|
|
loss=loss,
|
|
metrics={"accuracy": 0.8 + epoch * 0.05},
|
|
is_best=is_best,
|
|
)
|
|
print(f"✓ Saved checkpoint: {checkpoint_id}")
|
|
|
|
# List checkpoints
|
|
checkpoints = ckpt_manager.list_checkpoints()
|
|
print(f"✓ Found {len(checkpoints)} checkpoints")
|
|
|
|
# Test auto-resume
|
|
auto_resume = AutoResumeTrainer(ckpt_manager)
|
|
|
|
if auto_resume.should_resume():
|
|
new_model = SimpleModel()
|
|
new_optimizer = optim.Adam(new_model.parameters(), lr=0.001)
|
|
|
|
training_state = auto_resume.resume(new_model, new_optimizer)
|
|
print(f"✓ Auto-resumed from epoch {training_state['epoch']}, step {training_state['step']}")
|
|
|
|
# Load best checkpoint
|
|
best_model = SimpleModel()
|
|
best_state = ckpt_manager.load(best_model, checkpoint_id="best")
|
|
print(f"✓ Loaded best checkpoint: loss={best_state['loss']:.3f}")
|
|
|
|
# Cleanup old checkpoints
|
|
ckpt_manager.cleanup(keep_best=True, keep_latest=True)
|
|
print("✓ Cleaned up old checkpoints")
|
|
|
|
|
|
def test_integration():
|
|
"""Test integration of all components."""
|
|
print("\n=== Testing Full Integration ===")
|
|
|
|
# Setup components
|
|
coordinator = DistributedCoordinator(DistributedConfig())
|
|
metrics = MetricsCollector(log_dir="./test_integration/logs")
|
|
registry = ModelRegistry(base_path="./test_integration/registry")
|
|
ckpt_manager = CheckpointManager(checkpoint_dir="./test_integration/checkpoints")
|
|
monitor = TrainingMonitor(metrics)
|
|
|
|
# Initialize
|
|
coordinator.initialize()
|
|
|
|
# Create and wrap model
|
|
model = SimpleModel()
|
|
model = coordinator.wrap_model(model)
|
|
optimizer = optim.Adam(model.parameters(), lr=0.001)
|
|
|
|
# Training loop simulation
|
|
print("✓ Starting integrated training simulation...")
|
|
|
|
for epoch in range(2):
|
|
monitor.on_epoch_start(epoch)
|
|
|
|
# Simulate batches
|
|
for batch in range(3):
|
|
loss = 1.0 - (epoch * 0.2 + batch * 0.05)
|
|
monitor.on_batch_end(loss=loss, batch_size=32, learning_rate=0.001)
|
|
|
|
# Validation
|
|
val_loss = 0.8 - (epoch * 0.1)
|
|
val_metrics = {"loss": val_loss, "accuracy": 0.7 + epoch * 0.1}
|
|
is_best = monitor.on_validation(val_metrics)
|
|
|
|
# Save checkpoint
|
|
step = epoch * 100 + batch
|
|
ckpt_manager.save(
|
|
model=model,
|
|
optimizer=optimizer,
|
|
epoch=epoch,
|
|
step=step,
|
|
loss=val_loss,
|
|
metrics=val_metrics,
|
|
is_best=is_best,
|
|
)
|
|
|
|
monitor.on_epoch_end()
|
|
|
|
# Register final model
|
|
model_id = registry.register(
|
|
model=model,
|
|
name="integrated_model",
|
|
version="1.0.0",
|
|
architecture="feedforward",
|
|
metrics=val_metrics,
|
|
description="Model from integrated test",
|
|
)
|
|
|
|
registry.update_status(model_id, ModelStatus.PRODUCTION)
|
|
print(f"✓ Model registered and promoted to production: {model_id}")
|
|
|
|
# Cleanup
|
|
coordinator.cleanup()
|
|
metrics.close()
|
|
print("✓ Integration test completed successfully")
|
|
|
|
|
|
def main():
|
|
"""Run all tests."""
|
|
print("=" * 50)
|
|
print("Testing Gym Production Features")
|
|
print("=" * 50)
|
|
|
|
try:
|
|
test_distributed_training()
|
|
test_metrics_monitoring()
|
|
test_model_registry()
|
|
test_hyperparameter_search()
|
|
test_checkpoint_manager()
|
|
test_integration()
|
|
|
|
print("\n" + "=" * 50)
|
|
print("✅ All tests passed successfully!")
|
|
print("=" * 50)
|
|
|
|
except Exception as e:
|
|
print(f"\n❌ Test failed: {e}")
|
|
import traceback
|
|
traceback.print_exc()
|
|
sys.exit(1)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main() |