go/maintnotifications/e2e/scenario_template.go.example

package e2e

import (
	"context"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/redis/go-redis/v9"
	"github.com/redis/go-redis/v9/hitless"
)

// TestScenarioTemplate is a template for writing scenario tests
// Copy this file and rename it to scenario_your_test_name.go
func TestScenarioTemplate(t *testing.T) {
	if os.Getenv("E2E_SCENARIO_TESTS") != "true" {
		t.Skip("Scenario tests require E2E_SCENARIO_TESTS=true")
	}

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()

	// Step 1: Create client factory from configuration
	factory, err := CreateTestClientFactory("enterprise-cluster") // or "standalone0"
	if err != nil {
		t.Fatalf("Failed to create client factory: %v", err)
	}
	defer factory.DestroyAll()

	// Step 2: Create fault injector
	faultInjector, err := CreateTestFaultInjector()
	if err != nil {
		t.Fatalf("Failed to create fault injector: %v", err)
	}

	// Step 3: Create Redis client with hitless upgrades
	client, err := factory.Create("scenario-client", &CreateClientOptions{
		Protocol: 3,
		HitlessUpgradeConfig: &hitless.Config{
			Mode:           hitless.MaintNotificationsEnabled,
			HandoffTimeout: 30000, // 30 seconds
			RelaxedTimeout: 10000, // 10 seconds
			MaxWorkers:     20,
		},
		ClientName: "scenario-test-client",
	})
	if err != nil {
		t.Fatalf("Failed to create client: %v", err)
	}

	// Step 4: Verify initial connectivity
	err = client.Ping(ctx).Err()
	if err != nil {
		t.Fatalf("Failed to ping Redis: %v", err)
	}

	t.Log("Initial setup completed successfully")

	// Step 5: Start background operations (optional)
	stopCh := make(chan struct{})
	defer close(stopCh)

	go func() {
		counter := 0
		ticker := time.NewTicker(100 * time.Millisecond)
		defer ticker.Stop()

		for {
			select {
			case <-stopCh:
				return
			case <-ticker.C:
				key := fmt.Sprintf("test-key-%d", counter)
				value := fmt.Sprintf("test-value-%d", counter)

				err := client.Set(ctx, key, value, time.Minute).Err()
				if err != nil {
					t.Logf("Background operation failed: %v", err)
				}

				counter++
			}
		}
	}()

	// Step 6: Wait for baseline operations
	time.Sleep(5 * time.Second)

	// Step 7: Trigger fault injection scenario
	t.Log("Triggering fault injection scenario...")

	// Example: Cluster failover
	// resp, err := faultInjector.TriggerClusterFailover(ctx, "node-1", false)
	// if err != nil {
	//     t.Fatalf("Failed to trigger failover: %v", err)
	// }

	// Example: Network latency
	// nodes := []string{"localhost:7001", "localhost:7002"}
	// resp, err := faultInjector.SimulateNetworkLatency(ctx, nodes, 100*time.Millisecond, 20*time.Millisecond)
	// if err != nil {
	//     t.Fatalf("Failed to simulate latency: %v", err)
	// }

	// Example: Complex sequence
	// sequence := []SequenceAction{
	//     {
	//         Type: ActionNetworkLatency,
	//         Parameters: map[string]interface{}{
	//             "nodes":   []string{"localhost:7001"},
	//             "latency": "50ms",
	//         },
	//     },
	//     {
	//         Type: ActionClusterFailover,
	//         Parameters: map[string]interface{}{
	//             "node_id": "node-1",
	//             "force":   false,
	//         },
	//         Delay: 10 * time.Second,
	//     },
	// }
	// resp, err := faultInjector.ExecuteSequence(ctx, sequence)
	// if err != nil {
	//     t.Fatalf("Failed to execute sequence: %v", err)
	// }

	// Step 8: Wait for fault injection to complete
	// status, err := faultInjector.WaitForAction(ctx, resp.ActionID,
	//     WithMaxWaitTime(240*time.Second),
	//     WithPollInterval(2*time.Second))
	// if err != nil {
	//     t.Fatalf("Fault injection failed: %v", err)
	// }
	// t.Logf("Fault injection completed: %s", status.Status)

	// Step 9: Verify client remains operational during and after fault injection
	time.Sleep(10 * time.Second)

	err = client.Ping(ctx).Err()
	if err != nil {
		t.Errorf("Client not responsive after fault injection: %v", err)
	}

	// Step 10: Perform additional validation
	testKey := "validation-key"
	testValue := "validation-value"

	err = client.Set(ctx, testKey, testValue, time.Minute).Err()
	if err != nil {
		t.Errorf("Failed to set validation key: %v", err)
	}

	retrievedValue, err := client.Get(ctx, testKey).Result()
	if err != nil {
		t.Errorf("Failed to get validation key: %v", err)
	} else if retrievedValue != testValue {
		t.Errorf("Validation failed: expected %s, got %s", testValue, retrievedValue)
	}

	t.Log("Scenario test completed successfully")
}

// Helper functions for common scenario patterns

func performContinuousOperations(ctx context.Context, client redis.UniversalClient, workerID int, stopCh <-chan struct{}, errorCh chan<- error) {
	ticker := time.NewTicker(100 * time.Millisecond)
	defer ticker.Stop()

	counter := 0
	for {
		select {
		case <-stopCh:
			return
		case <-ticker.C:
			key := fmt.Sprintf("worker_%d_key_%d", workerID, counter)
			value := fmt.Sprintf("value_%d", counter)

			// Perform operation with timeout
			opCtx, cancel := context.WithTimeout(ctx, 5*time.Second)
			err := client.Set(opCtx, key, value, time.Minute).Err()
			cancel()

			if err != nil {
				select {
				case errorCh <- err:
				default:
				}
			}

			counter++
		}
	}
}

func validateClusterHealth(ctx context.Context, client redis.UniversalClient) error {
	// Basic connectivity test
	if err := client.Ping(ctx).Err(); err != nil {
		return fmt.Errorf("ping failed: %w", err)
	}

	// Test basic operations
	testKey := "health-check-key"
	testValue := "health-check-value"

	if err := client.Set(ctx, testKey, testValue, time.Minute).Err(); err != nil {
		return fmt.Errorf("set operation failed: %w", err)
	}

	retrievedValue, err := client.Get(ctx, testKey).Result()
	if err != nil {
		return fmt.Errorf("get operation failed: %w", err)
	}

	if retrievedValue != testValue {
		return fmt.Errorf("value mismatch: expected %s, got %s", testValue, retrievedValue)
	}

	// Clean up
	client.Del(ctx, testKey)

	return nil
}

func waitForStableOperations(ctx context.Context, client redis.UniversalClient, duration time.Duration) error {
	deadline := time.Now().Add(duration)
	ticker := time.NewTicker(1 * time.Second)
	defer ticker.Stop()

	for time.Now().Before(deadline) {
		select {
		case <-ctx.Done():
			return ctx.Err()
		case <-ticker.C:
			if err := validateClusterHealth(ctx, client); err != nil {
				return fmt.Errorf("cluster health check failed: %w", err)
			}
		}
	}

	return nil
}