insights/nodejs/tests/session-replay/recording-api/recording-api.integration.test.ts

/**
 * Integration tests for the Recording API
 *
 * Tests the full round-trip of:
 * 1. Encrypting block data
 * 2. Decrypting block data
 * 3. Handling deleted sessions (crypto shredding)
 * 4. HTTP endpoints via supertest
 * 5. S3 + decryption pipeline via localstack
 * 6. Cache invalidation on delete
 *
 * Includes tests with:
 * - MemoryKeyStore (always run)
 * - DynamoDB/KMS/S3 via Localstack (run by default, skip with LOCALSTACK_DISABLED=1)
 *
 * To skip localstack tests (e.g. when localstack isn't running):
 *   LOCALSTACK_DISABLED=1 pnpm jest src/session-replay/recording-api/recording-api.integration.test.ts
 */
import {
    CreateTableCommand,
    DeleteTableCommand,
    DescribeTableCommand,
    DynamoDBClient,
    ResourceNotFoundException,
} from '@aws-sdk/client-dynamodb'
import {
    CreateAliasCommand,
    CreateKeyCommand,
    DeleteAliasCommand,
    DescribeKeyCommand,
    KMSClient,
} from '@aws-sdk/client-kms'
import {
    CreateBucketCommand,
    DeleteBucketCommand,
    DeleteObjectCommand,
    ListObjectsV2Command,
    PutObjectCommand,
    S3Client,
} from '@aws-sdk/client-s3'
import { Server } from 'http'
import sodium from 'libsodium-wrappers'
import snappy from 'snappy'
import supertest from 'supertest'
import express from 'ultimate-express'

import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { PostgresRouter } from '~/common/utils/db/postgres'
import { parseJSON } from '~/common/utils/json-parse'
import { SodiumRecordingDecryptor, SodiumRecordingEncryptor } from '~/ingestion/pipelines/sessionreplay/shared/crypto'
import { DynamoDBKeyStore, MemoryKeyStore } from '~/ingestion/pipelines/sessionreplay/shared/keystore'
import { MemoryCachedKeyStore } from '~/ingestion/pipelines/sessionreplay/shared/keystore/cache'
import { ReplayEventsOutput, SessionFeaturesOutput } from '~/ingestion/pipelines/sessionreplay/shared/outputs'
import { RecordingApi } from '~/session-replay/recording-api/recording-api'
import { RecordingService } from '~/session-replay/recording-api/recording-service'
import { KeyStore, RecordingApiConfig, SessionKey, SessionKeyDeletedError } from '~/session-replay/recording-api/types'

const mockOutputs = {
    queueMessages: jest.fn().mockResolvedValue(undefined),
    produce: jest.fn().mockResolvedValue(undefined),
} as unknown as IngestionOutputs<ReplayEventsOutput | SessionFeaturesOutput>

// Localstack configuration
const LOCALSTACK_ENDPOINT = 'http://localhost:4566'
const KEYS_TABLE_NAME = 'session-recording-keys'
const KMS_KEY_ALIAS = 'alias/session-replay-master-key'
const shouldRunLocalstackTests = process.env.LOCALSTACK_DISABLED !== '1'

const isResourceNotFoundException = (error: unknown): boolean => {
    return (
        error instanceof ResourceNotFoundException ||
        (typeof error === 'object' &&
            error !== null &&
            'name' in error &&
            (error as { name?: unknown }).name === 'ResourceNotFoundException')
    )
}

// Helper functions shared across all tests
const createBlockData = async (events: unknown[]): Promise<Buffer> => {
    const jsonlContent = events.map((event) => JSON.stringify(['window1', event])).join('\n')
    return snappy.compress(jsonlContent)
}

const parseBlockData = async (data: Buffer): Promise<[string, unknown][]> => {
    const decompressed = await snappy.uncompress(data)
    return decompressed
        .toString()
        .trim()
        .split('\n')
        .map((line) => parseJSON(line))
}

describe('Recording API encryption integration', () => {
    beforeAll(async () => {
        await sodium.ready
    })

    /**
     * Shared test suite that runs against any KeyStore implementation
     */
    const runEncryptionTests = (
        getKeyStore: () => KeyStore,
        getEncryptor: () => SodiumRecordingEncryptor,
        getDecryptor: () => SodiumRecordingDecryptor
    ) => {
        describe('encrypt and decrypt round-trip', () => {
            it('should encrypt and decrypt block data correctly', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const sessionId = `test-session-${Date.now()}`
                const teamId = 42
                const originalEvents = [
                    { type: 2, data: { source: 1, snapshot: { html: '<div>Hello</div>' } } },
                    { type: 3, data: { source: 2, mutations: [{ id: 1 }] } },
                ]

                // Generate key first (simulates ingestion creating the key)
                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)

                const blockData = await createBlockData(originalEvents)
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                // Encrypted data should be different from original
                expect(encrypted.equals(blockData)).toBe(false)

                // Encrypted data should be larger (nonce + ciphertext overhead)
                expect(encrypted.length).toBeGreaterThan(blockData.length)

                // Decrypt
                const { data: decrypted, sessionState } = await decryptor.decryptBlock(sessionId, teamId, encrypted)

                // Decrypted data should match original
                expect(decrypted.equals(blockData)).toBe(true)
                expect(sessionState).toBe('ciphertext')

                // Parse and verify events
                const events = await parseBlockData(decrypted)
                expect(events).toHaveLength(2)
                expect(events[0][1]).toEqual(originalEvents[0])
                expect(events[1][1]).toEqual(originalEvents[1])
            })

            it('should use different nonces for each encryption', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const sessionId = `test-nonce-${Date.now()}`
                const teamId = 42
                const blockData = await createBlockData([{ type: 2, data: { content: 'same content' } }])

                // Generate key first
                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)

                const { data: encrypted1 } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)
                const { data: encrypted2 } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                // Extract nonces (first NONCEBYTES of each encrypted block)
                const nonce1 = encrypted1.subarray(0, sodium.crypto_secretbox_NONCEBYTES)
                const nonce2 = encrypted2.subarray(0, sodium.crypto_secretbox_NONCEBYTES)

                // Nonces should be different
                expect(Buffer.compare(nonce1, nonce2)).not.toBe(0)

                // Both should decrypt to same content
                const result1 = await decryptor.decryptBlock(sessionId, teamId, encrypted1)
                const result2 = await decryptor.decryptBlock(sessionId, teamId, encrypted2)
                expect(result1.data.equals(result2.data)).toBe(true)
            })

            it('should use different keys for different sessions', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const blockData = await createBlockData([{ type: 2, data: { content: 'same content' } }])
                const timestamp = Date.now()

                // Generate keys for both sessions
                const keyA = await keyStore.generateKey(`session-a-${timestamp}`, 42, 30)
                const keyB = await keyStore.generateKey(`session-b-${timestamp}`, 42, 30)

                const { data: encrypted1 } = encryptor.encryptBlockWithKey(
                    `session-a-${timestamp}`,
                    42,
                    blockData,
                    keyA
                )
                const { data: encrypted2 } = encryptor.encryptBlockWithKey(
                    `session-b-${timestamp}`,
                    42,
                    blockData,
                    keyB
                )

                // Decrypt with correct keys
                const result1 = await decryptor.decryptBlock(`session-a-${timestamp}`, 42, encrypted1)
                const result2 = await decryptor.decryptBlock(`session-b-${timestamp}`, 42, encrypted2)

                expect(result1.data.equals(blockData)).toBe(true)
                expect(result2.data.equals(blockData)).toBe(true)

                // Cross-decryption should fail (using wrong key)
                expect(() => {
                    decryptor.decryptBlockWithKey(`session-a-${timestamp}`, 42, encrypted1, keyB)
                }).toThrow()
            })

            it('should use different keys for different teams', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const sessionId = `same-session-${Date.now()}`
                const blockData = await createBlockData([{ type: 2, data: { content: 'content' } }])

                // Generate keys for both teams
                const key1 = await keyStore.generateKey(sessionId, 1, 30)
                const key2 = await keyStore.generateKey(sessionId, 2, 30)

                const { data: encryptedTeam1 } = encryptor.encryptBlockWithKey(sessionId, 1, blockData, key1)
                const { data: encryptedTeam2 } = encryptor.encryptBlockWithKey(sessionId, 2, blockData, key2)

                // Decrypt with correct team
                const resultTeam1 = await decryptor.decryptBlock(sessionId, 1, encryptedTeam1)
                const resultTeam2 = await decryptor.decryptBlock(sessionId, 2, encryptedTeam2)

                expect(resultTeam1.data.equals(blockData)).toBe(true)
                expect(resultTeam2.data.equals(blockData)).toBe(true)

                // Cross-team decryption should fail
                expect(() => {
                    decryptor.decryptBlockWithKey(sessionId, 1, encryptedTeam2, key1)
                }).toThrow()
            })
        })

        describe('crypto shredding (key deletion)', () => {
            it('should throw SessionKeyDeletedError after key is deleted', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const sessionId = `to-be-deleted-${Date.now()}`
                const teamId = 42
                const blockData = await createBlockData([{ type: 2, data: { secret: 'sensitive data' } }])

                // Generate key first
                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)

                // Encrypt
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                // Verify decryption works before deletion
                const resultBefore = await decryptor.decryptBlock(sessionId, teamId, encrypted)
                expect(resultBefore.data.equals(blockData)).toBe(true)

                // Delete the key
                const deleteResult = await keyStore.deleteKey(sessionId, teamId, '[email protected]')
                expect(deleteResult).toEqual({
                    status: 'deleted',
                    deletedAt: expect.any(Number),
                    deletedBy: '[email protected]',
                })

                // Verify decryption fails after deletion
                await expect(decryptor.decryptBlock(sessionId, teamId, encrypted)).rejects.toThrow(
                    SessionKeyDeletedError
                )
            })

            it('should include deletion timestamp in result', async () => {
                const keyStore = getKeyStore()

                const sessionId = `deleted-with-timestamp-${Date.now()}`
                const teamId = 42

                // Generate key first
                await keyStore.generateKey(sessionId, teamId, 30)

                // Delete and capture time (timestamps are in seconds)
                const beforeDelete = Math.floor(Date.now() / 1000)
                await keyStore.deleteKey(sessionId, teamId, '[email protected]')
                const afterDelete = Math.floor(Date.now() / 1000) + 1

                // Get key should return deleted state with timestamp
                const result = await keyStore.getKey(sessionId, teamId)
                expect(result.sessionState).toBe('deleted')
                expect(result.deletedAt).toBeGreaterThanOrEqual(beforeDelete)
                expect(result.deletedAt).toBeLessThanOrEqual(afterDelete)
            })

            it('should throw SessionKeyDeletedError when trying to encrypt deleted session', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()

                const sessionId = `deleted-before-encrypt-${Date.now()}`
                const teamId = 42
                const blockData = await createBlockData([{ type: 2, data: { content: 'content' } }])

                // Generate and delete key
                await keyStore.generateKey(sessionId, teamId, 30)
                await keyStore.deleteKey(sessionId, teamId, '[email protected]')

                // Encryption should fail
                await expect(encryptor.encryptBlock(sessionId, teamId, blockData)).rejects.toThrow(
                    SessionKeyDeletedError
                )
            })

            it('should create tombstone when deleting non-existent key', async () => {
                const keyStore = getKeyStore()

                const result = await keyStore.deleteKey(`non-existent-${Date.now()}`, 999, '[email protected]')
                expect(result.status).toBe('deleted')
                expect(result.deletedAt).toBeDefined()
            })

            it('should handle multiple sessions with selective deletion', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const timestamp = Date.now()
                const sessions = [
                    `keep-1-${timestamp}`,
                    `delete-1-${timestamp}`,
                    `keep-2-${timestamp}`,
                    `delete-2-${timestamp}`,
                ]
                const teamId = 42
                const encrypted: Record<string, Buffer> = {}

                // Generate keys and encrypt all sessions
                for (const sessionId of sessions) {
                    const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                    const blockData = await createBlockData([{ type: 2, data: { session: sessionId } }])
                    encrypted[sessionId] = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey).data
                }

                // Delete some sessions
                await keyStore.deleteKey(`delete-1-${timestamp}`, teamId, '[email protected]')
                await keyStore.deleteKey(`delete-2-${timestamp}`, teamId, '[email protected]')

                // Verify kept sessions are still decryptable
                for (const sessionId of [`keep-1-${timestamp}`, `keep-2-${timestamp}`]) {
                    const result = await decryptor.decryptBlock(sessionId, teamId, encrypted[sessionId])
                    const events = await parseBlockData(result.data)
                    expect(events[0][1]).toEqual({ type: 2, data: { session: sessionId } })
                }

                // Verify deleted sessions throw errors
                for (const sessionId of [`delete-1-${timestamp}`, `delete-2-${timestamp}`]) {
                    await expect(decryptor.decryptBlock(sessionId, teamId, encrypted[sessionId])).rejects.toThrow(
                        SessionKeyDeletedError
                    )
                }
            })
        })

        describe('large data handling', () => {
            it('should handle large block data', async () => {
                const keyStore = getKeyStore()
                const encryptor = getEncryptor()
                const decryptor = getDecryptor()

                const sessionId = `large-session-${Date.now()}`
                const teamId = 42

                // Generate key first
                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)

                // Create large events (100 events with substantial data)
                const largeEvents = Array.from({ length: 100 }, (_, i) => ({
                    type: 3,
                    data: {
                        source: 2,
                        mutations: Array.from({ length: 50 }, (_, j) => ({
                            id: i * 50 + j,
                            text: `This is mutation ${i * 50 + j} with some additional text to make it larger`,
                            attributes: { class: 'test-class', 'data-index': String(i * 50 + j) },
                        })),
                    },
                }))

                const blockData = await createBlockData(largeEvents)
                expect(blockData.length).toBeGreaterThan(50000) // Should be substantial

                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)
                const result = await decryptor.decryptBlock(sessionId, teamId, encrypted)

                expect(result.data.equals(blockData)).toBe(true)

                const events = await parseBlockData(result.data)
                expect(events).toHaveLength(100)
            })
        })
    }

    // Tests with MemoryKeyStore (always run)
    describe('with MemoryKeyStore', () => {
        let keyStore: MemoryKeyStore
        let encryptor: SodiumRecordingEncryptor
        let decryptor: SodiumRecordingDecryptor

        beforeEach(async () => {
            keyStore = new MemoryKeyStore()
            await keyStore.start()

            encryptor = new SodiumRecordingEncryptor(keyStore)
            await encryptor.start()

            decryptor = new SodiumRecordingDecryptor(keyStore)
            await decryptor.start()
        })

        runEncryptionTests(
            () => keyStore,
            () => encryptor,
            () => decryptor
        )

        describe('cleartext sessions', () => {
            it('should pass through data unchanged for cleartext sessions', async () => {
                const sessionId = 'cleartext-session'
                const teamId = 42
                const blockData = await createBlockData([{ type: 2, data: { content: 'not encrypted' } }])

                // Manually set a cleartext key
                const cleartextKey: SessionKey = {
                    plaintextKey: Buffer.alloc(0),
                    encryptedKey: Buffer.alloc(0),
                    sessionState: 'cleartext',
                }

                const encryptResult = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, cleartextKey)
                expect(encryptResult.data.equals(blockData)).toBe(true)
                expect(encryptResult.sessionState).toBe('cleartext')

                const decryptResult = decryptor.decryptBlockWithKey(sessionId, teamId, encryptResult.data, cleartextKey)
                expect(decryptResult.data.equals(blockData)).toBe(true)
                expect(decryptResult.sessionState).toBe('cleartext')
            })
        })
    })

    // Tests with DynamoDB/KMS via Localstack (run when LOCALSTACK_ENABLED=1)
    const describeLocalstack = shouldRunLocalstackTests ? describe : describe.skip

    describeLocalstack('with Localstack (DynamoDB + KMS)', () => {
        let dynamoDBClient: DynamoDBClient
        let kmsClient: KMSClient
        let kmsKeyId: string
        let keyStore: DynamoDBKeyStore
        let encryptor: SodiumRecordingEncryptor
        let decryptor: SodiumRecordingDecryptor

        async function setupKmsKey(): Promise<void> {
            try {
                const describeResult = await kmsClient.send(new DescribeKeyCommand({ KeyId: KMS_KEY_ALIAS }))
                if (describeResult.KeyMetadata?.KeyId) {
                    kmsKeyId = describeResult.KeyMetadata.KeyId
                    return
                }
            } catch {
                // Alias doesn't exist, create key and alias
            }

            const createKeyResult = await kmsClient.send(
                new CreateKeyCommand({
                    Description: 'Session replay master key for testing',
                    KeyUsage: 'ENCRYPT_DECRYPT',
                })
            )

            kmsKeyId = createKeyResult.KeyMetadata!.KeyId!

            await kmsClient.send(
                new CreateAliasCommand({
                    AliasName: KMS_KEY_ALIAS,
                    TargetKeyId: kmsKeyId,
                })
            )
        }

        async function setupDynamoDBTable(): Promise<void> {
            try {
                await dynamoDBClient.send(new DeleteTableCommand({ TableName: KEYS_TABLE_NAME }))
                await waitForTableDeletion()
            } catch {
                // Best-effort cleanup: table absence and Localstack's modeled errors vary in CI.
            }

            await dynamoDBClient.send(
                new CreateTableCommand({
                    TableName: KEYS_TABLE_NAME,
                    KeySchema: [
                        { AttributeName: 'session_id', KeyType: 'HASH' },
                        { AttributeName: 'team_id', KeyType: 'RANGE' },
                    ],
                    AttributeDefinitions: [
                        { AttributeName: 'session_id', AttributeType: 'S' },
                        { AttributeName: 'team_id', AttributeType: 'N' },
                    ],
                    BillingMode: 'PAY_PER_REQUEST',
                })
            )

            await waitForTableActive()
        }

        async function waitForTableActive(): Promise<void> {
            for (let i = 0; i < 30; i++) {
                try {
                    const result = await dynamoDBClient.send(new DescribeTableCommand({ TableName: KEYS_TABLE_NAME }))
                    if (result.Table?.TableStatus === 'ACTIVE') {
                        return
                    }
                } catch {
                    // Table not ready yet
                }
                await new Promise((resolve) => setTimeout(resolve, 1000))
            }
            throw new Error('Timeout waiting for DynamoDB table to become active')
        }

        async function waitForTableDeletion(): Promise<void> {
            for (let i = 0; i < 30; i++) {
                try {
                    await dynamoDBClient.send(new DescribeTableCommand({ TableName: KEYS_TABLE_NAME }))
                } catch (error) {
                    if (isResourceNotFoundException(error)) {
                        return
                    }
                }
                await new Promise((resolve) => setTimeout(resolve, 1000))
            }
            throw new Error('Timeout waiting for DynamoDB table deletion')
        }

        beforeAll(async () => {
            dynamoDBClient = new DynamoDBClient({
                endpoint: LOCALSTACK_ENDPOINT,
                region: 'us-east-1',
                credentials: { accessKeyId: 'test', secretAccessKey: 'test' },
            })

            kmsClient = new KMSClient({
                endpoint: LOCALSTACK_ENDPOINT,
                region: 'us-east-1',
                credentials: { accessKeyId: 'test', secretAccessKey: 'test' },
            })

            await setupKmsKey()
            await setupDynamoDBTable()
        }, 30000)

        afterAll(async () => {
            try {
                await dynamoDBClient.send(new DeleteTableCommand({ TableName: KEYS_TABLE_NAME }))
            } catch {
                // Ignore
            }

            try {
                await kmsClient.send(new DeleteAliasCommand({ AliasName: KMS_KEY_ALIAS }))
            } catch {
                // Ignore
            }

            dynamoDBClient.destroy()
            kmsClient.destroy()
        })

        beforeEach(async () => {
            keyStore = new DynamoDBKeyStore(dynamoDBClient, kmsClient)
            await keyStore.start()

            encryptor = new SodiumRecordingEncryptor(keyStore)
            await encryptor.start()

            decryptor = new SodiumRecordingDecryptor(keyStore)
            await decryptor.start()
        })

        runEncryptionTests(
            () => keyStore,
            () => encryptor,
            () => decryptor
        )

        describe('DynamoDBKeyStore specific', () => {
            it('should generate and retrieve an encrypted key via KMS', async () => {
                const sessionId = `kms-test-${Date.now()}`
                const teamId = 1

                const generatedKey = await keyStore.generateKey(sessionId, teamId, 30)

                expect(generatedKey.sessionState).toBe('ciphertext')
                expect(generatedKey.plaintextKey.length).toBe(sodium.crypto_secretbox_KEYBYTES)
                expect(generatedKey.encryptedKey.length).toBeGreaterThan(0)

                const retrievedKey = await keyStore.getKey(sessionId, teamId)

                expect(retrievedKey.sessionState).toBe('ciphertext')
                expect(retrievedKey.plaintextKey.equals(generatedKey.plaintextKey)).toBe(true)
                expect(retrievedKey.encryptedKey.equals(generatedKey.encryptedKey)).toBe(true)
            })

            it('should return already_deleted when deleting already deleted key', async () => {
                const sessionId = `double-delete-${Date.now()}`
                const teamId = 4

                await keyStore.generateKey(sessionId, teamId, 30)
                await keyStore.deleteKey(sessionId, teamId, '[email protected]')

                const result = await keyStore.deleteKey(sessionId, teamId, '[email protected]')
                expect(result.status).toBe('already_deleted')
                expect(result.deletedAt).toBeDefined()
            })

            it('should isolate keys between teams', async () => {
                const sessionId = `shared-session-${Date.now()}`
                const team1 = 10
                const team2 = 20

                const key1 = await keyStore.generateKey(sessionId, team1, 30)
                const key2 = await keyStore.generateKey(sessionId, team2, 30)

                expect(key1.plaintextKey.equals(key2.plaintextKey)).toBe(false)

                const retrieved1 = await keyStore.getKey(sessionId, team1)
                const retrieved2 = await keyStore.getKey(sessionId, team2)

                expect(retrieved1.plaintextKey.equals(key1.plaintextKey)).toBe(true)
                expect(retrieved2.plaintextKey.equals(key2.plaintextKey)).toBe(true)
            })

            it('should not affect other teams when deleting a key', async () => {
                const sessionId = `team-isolation-${Date.now()}`
                const team1 = 30
                const team2 = 40

                await keyStore.generateKey(sessionId, team1, 30)
                await keyStore.generateKey(sessionId, team2, 30)

                await keyStore.deleteKey(sessionId, team1, '[email protected]')

                const key1 = await keyStore.getKey(sessionId, team1)
                expect(key1.sessionState).toBe('deleted')

                const key2 = await keyStore.getKey(sessionId, team2)
                expect(key2.sessionState).toBe('ciphertext')
            })
        })

        describe('S3 + decryption pipeline', () => {
            let s3Client: S3Client
            let recordingService: RecordingService
            const S3_BUCKET = 'test-recording-bucket'
            const S3_PREFIX = 'session_recordings'

            beforeAll(async () => {
                s3Client = new S3Client({
                    endpoint: LOCALSTACK_ENDPOINT,
                    region: 'us-east-1',
                    credentials: { accessKeyId: 'test', secretAccessKey: 'test' },
                    forcePathStyle: true,
                })

                try {
                    await s3Client.send(new CreateBucketCommand({ Bucket: S3_BUCKET }))
                } catch {
                    // Bucket may already exist
                }
            }, 30000)

            afterAll(async () => {
                try {
                    const objects = await s3Client.send(new ListObjectsV2Command({ Bucket: S3_BUCKET }))
                    for (const obj of objects.Contents ?? []) {
                        await s3Client.send(new DeleteObjectCommand({ Bucket: S3_BUCKET, Key: obj.Key }))
                    }
                    await s3Client.send(new DeleteBucketCommand({ Bucket: S3_BUCKET }))
                } catch {
                    // Ignore cleanup errors
                }
                s3Client.destroy()
            })

            beforeEach(() => {
                recordingService = new RecordingService(s3Client, S3_BUCKET, S3_PREFIX, keyStore, decryptor)
            })

            it('should upload encrypted block to S3 and fetch it back decrypted', async () => {
                const sessionId = `s3-roundtrip-${Date.now()}`
                const teamId = 1
                const originalEvents = [
                    { type: 2, data: { source: 1, html: '<div>test</div>' } },
                    { type: 3, data: { source: 2, mutations: [{ id: 1 }] } },
                ]

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData(originalEvents)
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                const s3Key = `${S3_PREFIX}/30d/1700000000-abcdef0123456789`
                await s3Client.send(new PutObjectCommand({ Bucket: S3_BUCKET, Key: s3Key, Body: encrypted }))

                const result = await recordingService.getBlock({
                    sessionId,
                    teamId,
                    key: s3Key,
                    startByte: 0,
                    endByte: encrypted.length - 1,
                })

                expect(result.ok).toBe(true)
                if (result.ok) {
                    expect(result.data.equals(blockData)).toBe(true)
                    const events = await parseBlockData(result.data)
                    expect(events).toHaveLength(2)
                    expect(events[0][1]).toEqual(originalEvents[0])
                }
            })

            it('should return deleted result when key has been deleted', async () => {
                const sessionId = `s3-deleted-${Date.now()}`
                const teamId = 1

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData([{ type: 2, data: { content: 'secret' } }])
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                const s3Key = `${S3_PREFIX}/30d/1700000001-abcdef0123456789`
                await s3Client.send(new PutObjectCommand({ Bucket: S3_BUCKET, Key: s3Key, Body: encrypted }))

                await keyStore.deleteKey(sessionId, teamId, '[email protected]')

                const result = await recordingService.getBlock({
                    sessionId,
                    teamId,
                    key: s3Key,
                    startByte: 0,
                    endByte: encrypted.length - 1,
                })

                expect(result).toEqual(
                    expect.objectContaining({ ok: false, error: 'deleted', deletedAt: expect.any(Number) })
                )
            })
        })
    })

    // HTTP integration tests (always run, uses MemoryKeyStore)
    describe('HTTP endpoints', () => {
        let keyStore: MemoryKeyStore
        let encryptor: SodiumRecordingEncryptor
        let decryptor: SodiumRecordingDecryptor
        let app: express.Application
        let server: Server
        let mockS3Send: jest.Mock

        beforeEach(async () => {
            keyStore = new MemoryKeyStore()
            await keyStore.start()

            decryptor = new SodiumRecordingDecryptor(keyStore)
            await decryptor.start()

            encryptor = new SodiumRecordingEncryptor(keyStore)
            await encryptor.start()

            mockS3Send = jest.fn()
            const mockS3Client = { send: mockS3Send, destroy: jest.fn() } as unknown as S3Client

            const recordingService = new RecordingService(
                mockS3Client,
                'test-bucket',
                'session_recordings',
                keyStore,
                decryptor
            )

            const recordingApi = new RecordingApi({} as RecordingApiConfig, {} as PostgresRouter, mockOutputs)
            await recordingApi.start(recordingService)

            app = express()
            app.use('/', recordingApi.router())
            server = app.listen(0, () => {})
        })

        afterEach(() => {
            server.close()
        })

        describe('GET /block', () => {
            it('should return decrypted data with correct headers', async () => {
                const sessionId = 'http-session-1'
                const teamId = 1
                const originalEvents = [{ type: 2, data: { content: 'hello' } }]

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData(originalEvents)
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                mockS3Send.mockResolvedValue({
                    Body: { transformToByteArray: () => Promise.resolve(encrypted) },
                })

                const res = await supertest(app)
                    .get(`/v1/projects/${teamId}/recordings/${sessionId}/block`)
                    .query({
                        key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
                        start_byte: '0',
                        end_byte: String(encrypted.length - 1),
                    })

                expect(res.status).toBe(200)
                expect(res.headers['content-type']).toContain('application/octet-stream')
                expect(res.headers['cache-control']).toBe('public, max-age=2592000, immutable')
                expect(Buffer.from(res.body).equals(blockData)).toBe(true)
            })

            it('should return 410 for deleted session', async () => {
                const sessionId = 'http-deleted-session'
                const teamId = 1

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData([{ type: 2, data: { content: 'secret' } }])
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                await keyStore.deleteKey(sessionId, teamId, '[email protected]')

                mockS3Send.mockResolvedValue({
                    Body: { transformToByteArray: () => Promise.resolve(encrypted) },
                })

                const res = await supertest(app)
                    .get(`/v1/projects/${teamId}/recordings/${sessionId}/block`)
                    .query({
                        key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
                        start_byte: '0',
                        end_byte: String(encrypted.length - 1),
                    })

                expect(res.status).toBe(410)
                expect(res.body.error).toBe('recording_deleted')
                expect(res.body.deleted_at).toBeDefined()
                expect(res.body.deleted_by).toBe('[email protected]')
            })

            it('should return 404 when S3 returns no body', async () => {
                mockS3Send.mockResolvedValue({ Body: null })

                const res = await supertest(app).get('/v1/projects/1/recordings/session-1/block').query({
                    key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
                    start_byte: '0',
                    end_byte: '100',
                })

                expect(res.status).toBe(404)
                expect(res.body.error).toBe('Block not found')
            })

            it('should return decompressed JSONL when decompress=true', async () => {
                const sessionId = 'http-decompress-1'
                const teamId = 1
                const originalEvents = [
                    { type: 2, data: { content: 'hello' } },
                    { type: 3, data: { content: 'world' } },
                ]

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData(originalEvents)
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                mockS3Send.mockResolvedValue({
                    Body: { transformToByteArray: () => Promise.resolve(encrypted) },
                })

                const res = await supertest(app)
                    .get(`/v1/projects/${teamId}/recordings/${sessionId}/block`)
                    .query({
                        key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
                        start_byte: '0',
                        end_byte: String(encrypted.length - 1),
                        decompress: 'true',
                    })

                expect(res.status).toBe(200)
                expect(res.headers['content-type']).toContain('application/jsonl')
                expect(res.headers['cache-control']).toBe('public, max-age=2592000, immutable')

                const lines = res.text.trim().split('\n')
                expect(lines).toHaveLength(2)
                expect(parseJSON(lines[0])[1]).toEqual(originalEvents[0])
                expect(parseJSON(lines[1])[1]).toEqual(originalEvents[1])
            })

            it('should return compressed data when decompress is not set', async () => {
                const sessionId = 'http-no-decompress-1'
                const teamId = 1
                const originalEvents = [{ type: 2, data: { content: 'compressed' } }]

                const sessionKey = await keyStore.generateKey(sessionId, teamId, 30)
                const blockData = await createBlockData(originalEvents)
                const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

                mockS3Send.mockResolvedValue({
                    Body: { transformToByteArray: () => Promise.resolve(encrypted) },
                })

                const res = await supertest(app)
                    .get(`/v1/projects/${teamId}/recordings/${sessionId}/block`)
                    .query({
                        key: 'session_recordings/30d/1764634738680-3cca0f5d3c7cc7ee',
                        start_byte: '0',
                        end_byte: String(encrypted.length - 1),
                    })

                expect(res.status).toBe(200)
                expect(res.headers['content-type']).toContain('application/octet-stream')
                expect(Buffer.from(res.body).equals(blockData)).toBe(true)
            })
        })
    })

    // Cache invalidation tests (always run)
    describe('cache invalidation on delete', () => {
        it('should not serve stale cached key after deletion', async () => {
            const memoryKeyStore = new MemoryKeyStore()
            await memoryKeyStore.start()
            const cachedKeyStore = new MemoryCachedKeyStore(memoryKeyStore)

            const sessionId = `cache-invalidation-${Date.now()}`
            const teamId = 1

            await cachedKeyStore.generateKey(sessionId, teamId, 30)

            // Populate cache
            const cachedKey = await cachedKeyStore.getKey(sessionId, teamId)
            expect(cachedKey.sessionState).toBe('ciphertext')

            // Delete through cache layer
            await cachedKeyStore.deleteKey(sessionId, teamId, '[email protected]')

            // Next read must return deleted, not stale cached ciphertext
            const afterDelete = await cachedKeyStore.getKey(sessionId, teamId)
            expect(afterDelete.sessionState).toBe('deleted')
            expect(afterDelete.deletedAt).toBeDefined()
        })

        it('should throw SessionKeyDeletedError when decrypting after cached key deletion', async () => {
            const memoryKeyStore = new MemoryKeyStore()
            await memoryKeyStore.start()
            const cachedKeyStore = new MemoryCachedKeyStore(memoryKeyStore)

            const decryptor = new SodiumRecordingDecryptor(cachedKeyStore)
            await decryptor.start()

            const encryptor = new SodiumRecordingEncryptor(cachedKeyStore)
            await encryptor.start()

            const sessionId = `cache-decrypt-${Date.now()}`
            const teamId = 1

            const sessionKey = await cachedKeyStore.generateKey(sessionId, teamId, 30)
            const blockData = await createBlockData([{ type: 2, data: { content: 'cached' } }])
            const { data: encrypted } = encryptor.encryptBlockWithKey(sessionId, teamId, blockData, sessionKey)

            // Populate cache via decrypt
            const result = await decryptor.decryptBlock(sessionId, teamId, encrypted)
            expect(result.data.equals(blockData)).toBe(true)

            // Delete through cache
            await cachedKeyStore.deleteKey(sessionId, teamId, '[email protected]')

            // Decrypt must fail with SessionKeyDeletedError
            await expect(decryptor.decryptBlock(sessionId, teamId, encrypted)).rejects.toThrow(SessionKeyDeletedError)
        })

        it('should invalidate through nested cache layers', async () => {
            const memoryKeyStore = new MemoryKeyStore()
            await memoryKeyStore.start()

            // Simulate two-layer cache: outer memory cache wrapping inner memory cache
            const innerCache = new MemoryCachedKeyStore(memoryKeyStore)
            const outerCache = new MemoryCachedKeyStore(innerCache)

            const sessionId = `nested-cache-${Date.now()}`
            const teamId = 1

            await outerCache.generateKey(sessionId, teamId, 30)

            // Populate both cache layers
            const key = await outerCache.getKey(sessionId, teamId)
            expect(key.sessionState).toBe('ciphertext')

            // Delete through outer cache
            await outerCache.deleteKey(sessionId, teamId, '[email protected]')

            // All layers must return deleted
            const afterDelete = await outerCache.getKey(sessionId, teamId)
            expect(afterDelete.sessionState).toBe('deleted')
        })
    })
})