mirror of
https://github.com/hanzoai/stream.git
synced 2026-10-30 18:21:09 +00:00
zeekayandHanzo Dev
c3e231795c
This is the Kafka interface to Hanzo PubSub, not a product of its own. "Stream"
described the transport rather than the contract, and left two names for one
thing -- the repo was already redirecting hanzoai/kafka -> hanzoai/stream, so
imports and docs disagreed with the canonical URL.
The layering it belongs to:
hanzoai/tasks workloads and async workflows
|
hanzoai/pubsub the ONE event bus (JetStream-durable NATS)
|
hanzoai/kafka Kafka-wire adaptor (AMQP is the sibling seam)
|
hanzoai/datastore the ONE warehouse
Module path is now github.com/hanzoai/kafka; binary, image labels and docs say
Hanzo Kafka.
The word "stream" survives wherever it names the PROTOCOL rather than the
product -- JetStream stream names, ParseStreamName, the subject patterns in
pubsub/streams.go. Renaming those would have broken the wire contract to make a
brand consistent.
Unit tests pass. test/cluster and test/e2e still fail identically to before this
change: they shell out to an Apache Kafka distribution at ~/kafka_2.13-3.9.0/
that is not installed here, so they are unrelated to the rename.
Co-authored-by: Hanzo Dev <[email protected]>
82 lines
1.7 KiB
Go
82 lines
1.7 KiB
Go
package compress
|
|
|
|
import (
|
|
"bytes"
|
|
"compress/gzip"
|
|
"io"
|
|
"sync"
|
|
|
|
log "github.com/hanzoai/kafka/logging"
|
|
)
|
|
|
|
var (
|
|
gzipWriterPool = sync.Pool{
|
|
|
|
New: func() any {
|
|
return gzip.NewWriter(nil)
|
|
},
|
|
}
|
|
// we don't rely on New because gzip.NewReader
|
|
// can return an error
|
|
gzipReaderPool sync.Pool
|
|
)
|
|
|
|
// GzipCompressor implements Compressor interface
|
|
type GzipCompressor struct{} // TODO: handle the various GZIP levels (only default now)
|
|
|
|
// Compress takes in data and applies gzip to it
|
|
func (c *GzipCompressor) Compress(data []byte) ([]byte, error) {
|
|
|
|
var compressedData bytes.Buffer
|
|
gzipWriter := gzipWriterPool.Get().(*gzip.Writer)
|
|
defer gzipWriterPool.Put(gzipWriter)
|
|
gzipWriter.Reset(&compressedData)
|
|
// gzipWriter := gzip.NewWriter(&compressedData)
|
|
|
|
_, err := gzipWriter.Write(data)
|
|
if err != nil {
|
|
log.Error("Failed to compress data: %v", err)
|
|
return nil, err
|
|
|
|
}
|
|
|
|
err = gzipWriter.Close()
|
|
if err != nil {
|
|
log.Error("Failed to close GZIP writer: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
return compressedData.Bytes(), nil
|
|
}
|
|
|
|
// Decompress decompresses gzip-compressed data
|
|
func (c *GzipCompressor) Decompress(data []byte) ([]byte, error) {
|
|
var err error
|
|
gzipReader, found := gzipReaderPool.Get().(*gzip.Reader)
|
|
bytesReader := bytes.NewReader(data)
|
|
if found {
|
|
err = gzipReader.Reset(bytesReader)
|
|
} else {
|
|
// pool is empty, we init a new reader
|
|
gzipReader, err = gzip.NewReader(bytesReader)
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
defer gzipReaderPool.Put(gzipReader)
|
|
|
|
decompressedData, err := io.ReadAll(gzipReader)
|
|
if err != nil {
|
|
log.Error("Failed to decompress data: %v", err)
|
|
return nil, err
|
|
}
|
|
err = gzipReader.Close()
|
|
if err != nil {
|
|
log.Error("Failed to close GZIP reader: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
return decompressedData, nil
|
|
}
|