mirror of
https://github.com/hanzoai/kafka.git
synced 2026-10-30 14:48:25 +00:00
zeekayandHanzo Dev
c3e231795c
This is the Kafka interface to Hanzo PubSub, not a product of its own. "Stream"
described the transport rather than the contract, and left two names for one
thing -- the repo was already redirecting hanzoai/kafka -> hanzoai/stream, so
imports and docs disagreed with the canonical URL.
The layering it belongs to:
hanzoai/tasks workloads and async workflows
|
hanzoai/pubsub the ONE event bus (JetStream-durable NATS)
|
hanzoai/kafka Kafka-wire adaptor (AMQP is the sibling seam)
|
hanzoai/datastore the ONE warehouse
Module path is now github.com/hanzoai/kafka; binary, image labels and docs say
Hanzo Kafka.
The word "stream" survives wherever it names the PROTOCOL rather than the
product -- JetStream stream names, ParseStreamName, the subject patterns in
pubsub/streams.go. Renaming those would have broken the wire contract to make a
brand consistent.
Unit tests pass. test/cluster and test/e2e still fail identically to before this
change: they shell out to an Apache Kafka distribution at ~/kafka_2.13-3.9.0/
that is not installed here, so they are unrelated to the rename.
Co-authored-by: Hanzo Dev <[email protected]>
65 lines
1.7 KiB
Go
65 lines
1.7 KiB
Go
package compress
|
|
|
|
import (
|
|
"sync"
|
|
|
|
log "github.com/hanzoai/kafka/logging"
|
|
"github.com/klauspost/compress/zstd"
|
|
)
|
|
|
|
// ZSTDCompressor implements Compressor interface
|
|
type ZSTDCompressor struct{} // TODO: add compression levels
|
|
|
|
// Multiple instances of writers and readers can put pressure on the memory,
|
|
// we use a sync pool to reuse instance instead on letting the GC deal with them
|
|
var (
|
|
zstdWriterPool, zstdReaderPool sync.Pool
|
|
)
|
|
|
|
// Compress takes in data and applies ZSTD to it
|
|
func (c *ZSTDCompressor) Compress(data []byte) ([]byte, error) {
|
|
var res []byte
|
|
var err error
|
|
encoder, found := zstdWriterPool.Get().(*zstd.Encoder)
|
|
if !found {
|
|
// WithZeroFrames will encode 0 length input as full frames. This is needed for compatibility with zstandard usage
|
|
encoder, err = zstd.NewWriter(nil, zstd.WithZeroFrames(true))
|
|
}
|
|
defer zstdWriterPool.Put(encoder)
|
|
|
|
res = encoder.EncodeAll(data, nil)
|
|
log.Debug("encoder.EncodeAll \n%v \n%v ", data, res)
|
|
if err != nil {
|
|
log.Error("Failed to compress data: %v", err)
|
|
return nil, err
|
|
}
|
|
err = encoder.Close()
|
|
if err != nil {
|
|
log.Error("Failed to close ZSTD encoder: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
return res, nil
|
|
}
|
|
|
|
// Decompress decompresses ZSTD-compressed data
|
|
func (c *ZSTDCompressor) Decompress(data []byte) ([]byte, error) {
|
|
var err error
|
|
decoder, found := zstdReaderPool.Get().(*zstd.Decoder)
|
|
if !found {
|
|
// pool is empty, we init a new reader
|
|
decoder, err = zstd.NewReader(nil)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer zstdReaderPool.Put(decoder)
|
|
|
|
decompressedData, err := decoder.DecodeAll(data, nil)
|
|
if err != nil {
|
|
log.Error("Failed to decompress data: %v", err)
|
|
return nil, err
|
|
}
|
|
return decompressedData, nil
|
|
}
|