mirror of
https://github.com/hanzoai/stream.git
synced 2026-10-30 18:21:09 +00:00
zeekayandHanzo Dev
c3e231795c
This is the Kafka interface to Hanzo PubSub, not a product of its own. "Stream"
described the transport rather than the contract, and left two names for one
thing -- the repo was already redirecting hanzoai/kafka -> hanzoai/stream, so
imports and docs disagreed with the canonical URL.
The layering it belongs to:
hanzoai/tasks workloads and async workflows
|
hanzoai/pubsub the ONE event bus (JetStream-durable NATS)
|
hanzoai/kafka Kafka-wire adaptor (AMQP is the sibling seam)
|
hanzoai/datastore the ONE warehouse
Module path is now github.com/hanzoai/kafka; binary, image labels and docs say
Hanzo Kafka.
The word "stream" survives wherever it names the PROTOCOL rather than the
product -- JetStream stream names, ParseStreamName, the subject patterns in
pubsub/streams.go. Renaming those would have broken the wire contract to make a
brand consistent.
Unit tests pass. test/cluster and test/e2e still fail identically to before this
change: they shell out to an Apache Kafka distribution at ~/kafka_2.13-3.9.0/
that is not installed here, so they are unrelated to the rename.
Co-authored-by: Hanzo Dev <[email protected]>
63 lines
1.4 KiB
Go
63 lines
1.4 KiB
Go
package compress
|
|
|
|
import (
|
|
"bytes"
|
|
"sync"
|
|
|
|
log "github.com/hanzoai/kafka/logging"
|
|
"github.com/pierrec/lz4/v4"
|
|
)
|
|
|
|
// LZ4Compressor implements Compressor interface
|
|
type LZ4Compressor struct{}
|
|
|
|
// Multiple instances of writers and readers can put pressure on the memory,
|
|
// we use a sync pool to reuse instance instead on letting the GC deal with them
|
|
var (
|
|
LZ4WriterPool = sync.Pool{
|
|
New: func() any {
|
|
return lz4.NewWriter(nil)
|
|
},
|
|
}
|
|
LZ4ReaderPool = sync.Pool{
|
|
New: func() any {
|
|
return lz4.NewReader(nil)
|
|
},
|
|
}
|
|
)
|
|
|
|
// Compress takes in data and applies LZ4 to it
|
|
func (c *LZ4Compressor) Compress(data []byte) ([]byte, error) {
|
|
var buf bytes.Buffer
|
|
writer := LZ4WriterPool.Get().(*lz4.Writer)
|
|
writer.Reset(&buf)
|
|
defer LZ4WriterPool.Put(writer)
|
|
_, err := writer.Write(data)
|
|
if err != nil {
|
|
log.Error("Failed to compress data: %v", err)
|
|
}
|
|
err = writer.Close()
|
|
if err != nil {
|
|
log.Error("Failed to close LZ4 writer: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
return buf.Bytes(), nil
|
|
}
|
|
|
|
// Decompress decompresses LZ4-compressed data
|
|
func (c *LZ4Compressor) Decompress(data []byte) ([]byte, error) {
|
|
reader := LZ4ReaderPool.Get().(*lz4.Reader)
|
|
reader.Reset(bytes.NewReader(data))
|
|
defer LZ4ReaderPool.Put(reader)
|
|
|
|
var decompressedData bytes.Buffer
|
|
_, err := decompressedData.ReadFrom(reader)
|
|
if err != nil {
|
|
log.Error("Failed to decompress data: %v", err)
|
|
return nil, err
|
|
}
|
|
|
|
return decompressedData.Bytes(), nil
|
|
}
|