mirror of
https://github.com/hanzoai/stream.git
synced 2026-10-30 18:21:09 +00:00
The 2026-07 insights outage, decomplected into its three real defects: - CommitOffset rejected any offset above 2^50 as 'corruption' and GetCommittedOffset purged stored ones. Hanzo PubSub allocates sequences from a sparse e18 space, so every legitimate commit was refused (librdkafka's 'Unknown broker error') and group state self-destructed. Offsets are opaque int64s now; only negative is invalid. - Anything published raw onto a kafka-* subject was served verbatim as a record batch; the client read its bytes 8-11 as a batch length and died with MSG_SIZE_TOO_LARGE, produce died parsing it, and binary search went blind — one stray message wedged the partition until a stream purge. walkBatches() is now the single gate: produce refuses chains that do not walk (INVALID_RECORD), fetch and bounds skip stored messages that do not. - Sequence arithmetic assumed density. All log positions now come from the stamped batch headers via partitionBounds/findRecordSet, reading through NextMessage (by-start-sequence subscription) so holes and e18-sparse sequence spaces behave identically to dense ones. Also: fetch answers OFFSET_OUT_OF_RANGE instead of stalling consumers on dead positions; per-partition errors replace the top-level goto; the response encoder grows to fit instead of silently truncating past 64KiB; request frames are capped at 128MiB so a port scan cannot OOM the broker; OffsetFetch v8 no longer misparses (member fields are v9+) and its response drops a field the schema never had. Co-authored-by: Hanzo Dev <[email protected]>
225 lines
6.8 KiB
Go
225 lines
6.8 KiB
Go
package protocol
|
|
|
|
// InitProducerIDRequest represents the request for producer ID initialization.
|
|
type InitProducerIDRequest struct {
|
|
TransactionalID string `kafka:"CompactString"`
|
|
TransactionTimeoutMs uint32
|
|
ProducerID uint64
|
|
ProducerEpoch uint16
|
|
}
|
|
|
|
// InitProducerIDResponse represents the response to a producer ID initialization request.
|
|
type InitProducerIDResponse struct {
|
|
ThrottleTimeMs uint32
|
|
ErrorCode uint16
|
|
ProducerID uint64
|
|
ProducerEpoch uint16
|
|
}
|
|
|
|
// SyncGroupRequest represents the details of a SyncGroupRequest.
|
|
type SyncGroupRequest struct {
|
|
GroupID string `kafka:"CompactString"`
|
|
GenerationID uint32
|
|
MemberID string `kafka:"CompactString"`
|
|
GroupInstanceID string `kafka:"CompactNullableString"`
|
|
ProtocolType string `kafka:"CompactNullableString"`
|
|
ProtocolName string `kafka:"CompactNullableString"`
|
|
Assignments []SyncGroupRequestMember
|
|
}
|
|
|
|
// SyncGroupRequestMember represents a member and its assignment in the SyncGroupRequest.
|
|
type SyncGroupRequestMember struct {
|
|
MemberID string `kafka:"CompactString"`
|
|
Assignment []byte
|
|
}
|
|
|
|
// SyncGroupResponse represents a SyncGroup
|
|
type SyncGroupResponse struct {
|
|
ThrottleTimeMs uint32
|
|
ErrorCode uint16
|
|
ProtocolType string `kafka:"CompactString"`
|
|
ProtocolName string `kafka:"CompactString"`
|
|
AssignmentBytes []byte
|
|
}
|
|
|
|
// HeartbeatResponse represents a HearBeat
|
|
type HeartbeatResponse struct {
|
|
ThrottleTimeMs uint32
|
|
ErrorCode uint16
|
|
}
|
|
|
|
// JoinGroupRequest represents a JoinGroup request
|
|
type JoinGroupRequest struct {
|
|
GroupID string `kafka:"CompactString"`
|
|
SessionTimeoutMs uint32
|
|
RebalanceTimeoutMs uint32
|
|
MemberID string `kafka:"CompactString"`
|
|
GroupInstanceID string `kafka:"CompactNullableString"` // Nullable fields are represented as empty strings if not set
|
|
ProtocolType string `kafka:"CompactString"`
|
|
Protocols []JoinGroupRequestProtocol
|
|
Reason string `kafka:"CompactNullableString"` // Nullable fields are represented as empty strings if not set
|
|
}
|
|
|
|
// JoinGroupRequestProtocol represents a protocol in JoinGroupRequest
|
|
type JoinGroupRequestProtocol struct {
|
|
Name string `kafka:"CompactString"`
|
|
Metadata []byte
|
|
}
|
|
|
|
// JoinGroupResponseMember represents a member in a join group response.
|
|
type JoinGroupResponseMember struct {
|
|
MemberID string `kafka:"CompactString"`
|
|
GroupInstanceID string `kafka:"CompactString"`
|
|
Metadata []byte
|
|
}
|
|
|
|
// JoinGroupResponse represents the response to a join group request.
|
|
type JoinGroupResponse struct {
|
|
ThrottleTimeMS uint32
|
|
ErrorCode uint16
|
|
GenerationID uint32
|
|
ProtocolType string `kafka:"CompactString"`
|
|
ProtocolName string `kafka:"CompactString"`
|
|
Leader string `kafka:"CompactString"`
|
|
SkipAssignment bool
|
|
MemberID string `kafka:"CompactString"`
|
|
Members []JoinGroupResponseMember
|
|
}
|
|
|
|
// OffsetFetchRequest represents the offset fetch request.
|
|
type OffsetFetchRequest struct {
|
|
Groups []OffsetFetchRequestGroup
|
|
RequiresStable bool
|
|
}
|
|
|
|
// OffsetFetchRequestGroup represents a group in OffsetFetchRequest
|
|
type OffsetFetchRequestGroup struct {
|
|
GroupID string `kafka:"CompactString"`
|
|
MemberID string `kafka:"CompactNullableString"`
|
|
MemberEpoch uint32
|
|
Topics []OffsetFetchRequestTopic
|
|
}
|
|
|
|
// OffsetFetchRequestTopic represents a topic in OffsetFetchRequestGroup
|
|
type OffsetFetchRequestTopic struct {
|
|
Name string `kafka:"CompactString"`
|
|
PartitionIndexes []uint32
|
|
}
|
|
|
|
// OffsetFetchResponse represents the response to an offset fetch request.
|
|
type OffsetFetchResponse struct {
|
|
ThrottleTimeMs uint32
|
|
Groups []OffsetFetchGroup
|
|
}
|
|
|
|
// OffsetFetchGroup represents a group in an offset fetch response.
|
|
type OffsetFetchGroup struct {
|
|
GroupID string `kafka:"CompactString"`
|
|
Topics []OffsetFetchTopic
|
|
ErrorCode uint16
|
|
}
|
|
|
|
// OffsetFetchTopic represents a topic in an offset fetch response.
|
|
type OffsetFetchTopic struct {
|
|
Name string
|
|
Partitions []OffsetFetchPartition
|
|
}
|
|
|
|
// OffsetFetchPartition represents a partition in an offset fetch response.
|
|
type OffsetFetchPartition struct {
|
|
PartitionIndex uint32
|
|
CommittedOffset uint64
|
|
CommittedLeaderEpoch uint32
|
|
Metadata string
|
|
ErrorCode uint16
|
|
}
|
|
|
|
// ListOffsetsRequest represents a request to list offsets for specific partitions.
|
|
type ListOffsetsRequest struct {
|
|
ReplicaID uint32
|
|
IsolationLevel uint8
|
|
Topics []ListOffsetsRequestTopic
|
|
}
|
|
|
|
// ListOffsetsRequestTopic represents a topic in a list offsets request.
|
|
type ListOffsetsRequestTopic struct {
|
|
Name string
|
|
Partitions []ListOffsetsRequestPartition
|
|
}
|
|
|
|
// ListOffsetsRequestPartition represents a partition in a list offsets request.
|
|
type ListOffsetsRequestPartition struct {
|
|
PartitionIndex uint32
|
|
CurrentLeaderEpoch uint32
|
|
Timestamp uint64
|
|
}
|
|
|
|
// Constants for list offsets timestamps.
|
|
var (
|
|
ListOffsetsEarliestTimestamp = -2
|
|
ListOffsetsLatestTimestamp = -1
|
|
ListOffsetsMaxTimestamp = -3
|
|
)
|
|
|
|
// ListOffsetsResponse represents the response to a list offsets request.
|
|
type ListOffsetsResponse struct {
|
|
ThrottleTimeMs uint32
|
|
Topics []ListOffsetsResponseTopic
|
|
}
|
|
|
|
// ListOffsetsResponseTopic represents a topic in a list offsets response.
|
|
type ListOffsetsResponseTopic struct {
|
|
Name string `kafka:"CompactString"`
|
|
Partitions []ListOffsetsResponsePartition
|
|
}
|
|
|
|
// ListOffsetsResponsePartition represents a partition in a list offsets response.
|
|
type ListOffsetsResponsePartition struct {
|
|
PartitionIndex uint32
|
|
ErrorCode uint16
|
|
Timestamp uint64
|
|
Offset uint64
|
|
LeaderEpoch uint32
|
|
}
|
|
|
|
// OffsetCommitRequest represents a request to commit offsets.
|
|
type OffsetCommitRequest struct {
|
|
GroupID string
|
|
GenerationIDOrMemberEpoch uint32
|
|
MemberID string
|
|
GroupInstanceID string
|
|
Topics []OffsetCommitRequestTopic
|
|
}
|
|
|
|
// OffsetCommitRequestTopic represents a topic in an offset commit request.
|
|
type OffsetCommitRequestTopic struct {
|
|
Name string
|
|
Partitions []OffsetCommitRequestPartition
|
|
}
|
|
|
|
// OffsetCommitRequestPartition represents a partition in an offset commit request.
|
|
type OffsetCommitRequestPartition struct {
|
|
PartitionIndex uint32
|
|
CommittedOffset uint64
|
|
CommittedLeaderEpoch uint32
|
|
CommittedMetadata string `kafka:"CompactString"`
|
|
}
|
|
|
|
// OffsetCommitResponse represents the response to an offset commit request.
|
|
type OffsetCommitResponse struct {
|
|
ThrottleTimeMs uint32
|
|
Topics []OffsetCommitResponseTopic
|
|
}
|
|
|
|
// OffsetCommitResponseTopic represents a topic in an offset commit response.
|
|
type OffsetCommitResponseTopic struct {
|
|
Name string `kafka:"CompactString"`
|
|
Partitions []OffsetCommitResponsePartition
|
|
}
|
|
|
|
// OffsetCommitResponsePartition represents a partition in an offset commit response.
|
|
type OffsetCommitResponsePartition struct {
|
|
PartitionIndex uint32
|
|
ErrorCode uint16
|
|
}
|