Files
zeekayandHanzo Dev e58c08bce5 broker: offsets from batch headers only — sparse-sequence correct, poison-proof
The 2026-07 insights outage, decomplected into its three real defects:

- CommitOffset rejected any offset above 2^50 as 'corruption' and
  GetCommittedOffset purged stored ones. Hanzo PubSub allocates sequences
  from a sparse e18 space, so every legitimate commit was refused
  (librdkafka's 'Unknown broker error') and group state self-destructed.
  Offsets are opaque int64s now; only negative is invalid.

- Anything published raw onto a kafka-* subject was served verbatim as a
  record batch; the client read its bytes 8-11 as a batch length and died
  with MSG_SIZE_TOO_LARGE, produce died parsing it, and binary search went
  blind — one stray message wedged the partition until a stream purge.
  walkBatches() is now the single gate: produce refuses chains that do not
  walk (INVALID_RECORD), fetch and bounds skip stored messages that do not.

- Sequence arithmetic assumed density. All log positions now come from the
  stamped batch headers via partitionBounds/findRecordSet, reading through
  NextMessage (by-start-sequence subscription) so holes and e18-sparse
  sequence spaces behave identically to dense ones.

Also: fetch answers OFFSET_OUT_OF_RANGE instead of stalling consumers on
dead positions; per-partition errors replace the top-level goto; the
response encoder grows to fit instead of silently truncating past 64KiB;
request frames are capped at 128MiB so a port scan cannot OOM the broker;
OffsetFetch v8 no longer misparses (member fields are v9+) and its response
drops a field the schema never had.

Co-authored-by: Hanzo Dev <[email protected]>
2026-07-30 06:26:08 -07:00

225 lines
6.8 KiB
Go

package protocol
// InitProducerIDRequest represents the request for producer ID initialization.
type InitProducerIDRequest struct {
TransactionalID string `kafka:"CompactString"`
TransactionTimeoutMs uint32
ProducerID uint64
ProducerEpoch uint16
}
// InitProducerIDResponse represents the response to a producer ID initialization request.
type InitProducerIDResponse struct {
ThrottleTimeMs uint32
ErrorCode uint16
ProducerID uint64
ProducerEpoch uint16
}
// SyncGroupRequest represents the details of a SyncGroupRequest.
type SyncGroupRequest struct {
GroupID string `kafka:"CompactString"`
GenerationID uint32
MemberID string `kafka:"CompactString"`
GroupInstanceID string `kafka:"CompactNullableString"`
ProtocolType string `kafka:"CompactNullableString"`
ProtocolName string `kafka:"CompactNullableString"`
Assignments []SyncGroupRequestMember
}
// SyncGroupRequestMember represents a member and its assignment in the SyncGroupRequest.
type SyncGroupRequestMember struct {
MemberID string `kafka:"CompactString"`
Assignment []byte
}
// SyncGroupResponse represents a SyncGroup
type SyncGroupResponse struct {
ThrottleTimeMs uint32
ErrorCode uint16
ProtocolType string `kafka:"CompactString"`
ProtocolName string `kafka:"CompactString"`
AssignmentBytes []byte
}
// HeartbeatResponse represents a HearBeat
type HeartbeatResponse struct {
ThrottleTimeMs uint32
ErrorCode uint16
}
// JoinGroupRequest represents a JoinGroup request
type JoinGroupRequest struct {
GroupID string `kafka:"CompactString"`
SessionTimeoutMs uint32
RebalanceTimeoutMs uint32
MemberID string `kafka:"CompactString"`
GroupInstanceID string `kafka:"CompactNullableString"` // Nullable fields are represented as empty strings if not set
ProtocolType string `kafka:"CompactString"`
Protocols []JoinGroupRequestProtocol
Reason string `kafka:"CompactNullableString"` // Nullable fields are represented as empty strings if not set
}
// JoinGroupRequestProtocol represents a protocol in JoinGroupRequest
type JoinGroupRequestProtocol struct {
Name string `kafka:"CompactString"`
Metadata []byte
}
// JoinGroupResponseMember represents a member in a join group response.
type JoinGroupResponseMember struct {
MemberID string `kafka:"CompactString"`
GroupInstanceID string `kafka:"CompactString"`
Metadata []byte
}
// JoinGroupResponse represents the response to a join group request.
type JoinGroupResponse struct {
ThrottleTimeMS uint32
ErrorCode uint16
GenerationID uint32
ProtocolType string `kafka:"CompactString"`
ProtocolName string `kafka:"CompactString"`
Leader string `kafka:"CompactString"`
SkipAssignment bool
MemberID string `kafka:"CompactString"`
Members []JoinGroupResponseMember
}
// OffsetFetchRequest represents the offset fetch request.
type OffsetFetchRequest struct {
Groups []OffsetFetchRequestGroup
RequiresStable bool
}
// OffsetFetchRequestGroup represents a group in OffsetFetchRequest
type OffsetFetchRequestGroup struct {
GroupID string `kafka:"CompactString"`
MemberID string `kafka:"CompactNullableString"`
MemberEpoch uint32
Topics []OffsetFetchRequestTopic
}
// OffsetFetchRequestTopic represents a topic in OffsetFetchRequestGroup
type OffsetFetchRequestTopic struct {
Name string `kafka:"CompactString"`
PartitionIndexes []uint32
}
// OffsetFetchResponse represents the response to an offset fetch request.
type OffsetFetchResponse struct {
ThrottleTimeMs uint32
Groups []OffsetFetchGroup
}
// OffsetFetchGroup represents a group in an offset fetch response.
type OffsetFetchGroup struct {
GroupID string `kafka:"CompactString"`
Topics []OffsetFetchTopic
ErrorCode uint16
}
// OffsetFetchTopic represents a topic in an offset fetch response.
type OffsetFetchTopic struct {
Name string
Partitions []OffsetFetchPartition
}
// OffsetFetchPartition represents a partition in an offset fetch response.
type OffsetFetchPartition struct {
PartitionIndex uint32
CommittedOffset uint64
CommittedLeaderEpoch uint32
Metadata string
ErrorCode uint16
}
// ListOffsetsRequest represents a request to list offsets for specific partitions.
type ListOffsetsRequest struct {
ReplicaID uint32
IsolationLevel uint8
Topics []ListOffsetsRequestTopic
}
// ListOffsetsRequestTopic represents a topic in a list offsets request.
type ListOffsetsRequestTopic struct {
Name string
Partitions []ListOffsetsRequestPartition
}
// ListOffsetsRequestPartition represents a partition in a list offsets request.
type ListOffsetsRequestPartition struct {
PartitionIndex uint32
CurrentLeaderEpoch uint32
Timestamp uint64
}
// Constants for list offsets timestamps.
var (
ListOffsetsEarliestTimestamp = -2
ListOffsetsLatestTimestamp = -1
ListOffsetsMaxTimestamp = -3
)
// ListOffsetsResponse represents the response to a list offsets request.
type ListOffsetsResponse struct {
ThrottleTimeMs uint32
Topics []ListOffsetsResponseTopic
}
// ListOffsetsResponseTopic represents a topic in a list offsets response.
type ListOffsetsResponseTopic struct {
Name string `kafka:"CompactString"`
Partitions []ListOffsetsResponsePartition
}
// ListOffsetsResponsePartition represents a partition in a list offsets response.
type ListOffsetsResponsePartition struct {
PartitionIndex uint32
ErrorCode uint16
Timestamp uint64
Offset uint64
LeaderEpoch uint32
}
// OffsetCommitRequest represents a request to commit offsets.
type OffsetCommitRequest struct {
GroupID string
GenerationIDOrMemberEpoch uint32
MemberID string
GroupInstanceID string
Topics []OffsetCommitRequestTopic
}
// OffsetCommitRequestTopic represents a topic in an offset commit request.
type OffsetCommitRequestTopic struct {
Name string
Partitions []OffsetCommitRequestPartition
}
// OffsetCommitRequestPartition represents a partition in an offset commit request.
type OffsetCommitRequestPartition struct {
PartitionIndex uint32
CommittedOffset uint64
CommittedLeaderEpoch uint32
CommittedMetadata string `kafka:"CompactString"`
}
// OffsetCommitResponse represents the response to an offset commit request.
type OffsetCommitResponse struct {
ThrottleTimeMs uint32
Topics []OffsetCommitResponseTopic
}
// OffsetCommitResponseTopic represents a topic in an offset commit response.
type OffsetCommitResponseTopic struct {
Name string `kafka:"CompactString"`
Partitions []OffsetCommitResponsePartition
}
// OffsetCommitResponsePartition represents a partition in an offset commit response.
type OffsetCommitResponsePartition struct {
PartitionIndex uint32
ErrorCode uint16
}