mirror of
https://github.com/luxfi/zapdb.git
synced 2026-10-30 15:12:20 +00:00
Ibrahim Jarif
58a11fabb6
This commit adds support for compression/encryption in the background.
All incoming writes are saved to a common buffer and then worker go
routine pick up each block (a block is just a start and end point in
the main buffer). The worker goroutines also fix the table start and
end points. We fix the interleaving space (which might be caused because
the block was compressed) in the builder.Finish() call. All the blocks
are copied over to their correct position.
Master branch vs This commit (new) Benchmarks.
```
benchstat master.txt new.txt
name old time/op new time/op delta (-ve is better. Reduction in time)
no_compression-16 177ms ± 1% 177ms ± 2% ~ (p=0.690 n=5+5)
encryption-16 313ms ± 4% 245ms ± 3% -21.69% (p=0.008 n=5+5)
zstd_compression/level_1-16 330ms ± 1% 227ms ± 2% -31.22% (p=0.008 n=5+5)
zstd_compression/level_3-16 345ms ± 2% 227ms ± 3% -34.39% (p=0.008 n=5+5)
zstd_compression/level_15-16 10.7s ± 1% 1.2s ± 0% -88.96% (p=0.008 n=5+5)
name old speed new speed delta (+ve is better. Speed improvement)
no_compression-16 471MB/s ± 1% 471MB/s ± 2% ~ (p=0.690 n=5+5)
encryption-16 266MB/s ± 4% 340MB/s ± 2% +27.67% (p=0.008 n=5+5)
zstd_compression/level_1-16 252MB/s ± 1% 367MB/s ± 2% +45.40% (p=0.008 n=5+5)
zstd_compression/level_3-16 241MB/s ± 2% 367MB/s ± 3% +52.44% (p=0.008 n=5+5)
zstd_compression/level_15-16 7.80MB/s ± 1% 70.62MB/s ± 0% +805.33% (p=0.008 n=5+5)
```
Please look at https://github.com/dgraph-io/badger/pull/1227#issue-378982789
for the code used for benchmarking.
(cherry picked from commit b13b927102)
100 lines
2.4 KiB
Go
100 lines
2.4 KiB
Go
/*
|
|
* Copyright 2017 Dgraph Labs, Inc. and Contributors
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package y
|
|
|
|
import (
|
|
"bytes"
|
|
"encoding/binary"
|
|
)
|
|
|
|
// ValueStruct represents the value info that can be associated with a key, but also the internal
|
|
// Meta field.
|
|
type ValueStruct struct {
|
|
Meta byte
|
|
UserMeta byte
|
|
ExpiresAt uint64
|
|
Value []byte
|
|
|
|
Version uint64 // This field is not serialized. Only for internal usage.
|
|
}
|
|
|
|
func sizeVarint(x uint64) (n int) {
|
|
for {
|
|
n++
|
|
x >>= 7
|
|
if x == 0 {
|
|
break
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
// EncodedSize is the size of the ValueStruct when encoded
|
|
func (v *ValueStruct) EncodedSize() uint32 {
|
|
sz := len(v.Value) + 2 // meta, usermeta.
|
|
if v.ExpiresAt == 0 {
|
|
return uint32(sz + 1)
|
|
}
|
|
|
|
enc := sizeVarint(v.ExpiresAt)
|
|
return uint32(sz + enc)
|
|
}
|
|
|
|
// Decode uses the length of the slice to infer the length of the Value field.
|
|
func (v *ValueStruct) Decode(b []byte) {
|
|
v.Meta = b[0]
|
|
v.UserMeta = b[1]
|
|
var sz int
|
|
v.ExpiresAt, sz = binary.Uvarint(b[2:])
|
|
v.Value = b[2+sz:]
|
|
}
|
|
|
|
// Encode expects a slice of length at least v.EncodedSize().
|
|
func (v *ValueStruct) Encode(b []byte) uint32 {
|
|
b[0] = v.Meta
|
|
b[1] = v.UserMeta
|
|
sz := binary.PutUvarint(b[2:], v.ExpiresAt)
|
|
n := copy(b[2+sz:], v.Value)
|
|
return uint32(2 + sz + n)
|
|
}
|
|
|
|
// EncodeTo should be kept in sync with the Encode function above. The reason
|
|
// this function exists is to avoid creating byte arrays per key-value pair in
|
|
// table/builder.go.
|
|
func (v *ValueStruct) EncodeTo(buf *bytes.Buffer) {
|
|
buf.WriteByte(v.Meta)
|
|
buf.WriteByte(v.UserMeta)
|
|
var enc [binary.MaxVarintLen64]byte
|
|
sz := binary.PutUvarint(enc[:], v.ExpiresAt)
|
|
|
|
buf.Write(enc[:sz])
|
|
buf.Write(v.Value)
|
|
}
|
|
|
|
// Iterator is an interface for a basic iterator.
|
|
type Iterator interface {
|
|
Next()
|
|
Rewind()
|
|
Seek(key []byte)
|
|
Key() []byte
|
|
Value() ValueStruct
|
|
Valid() bool
|
|
|
|
// All iterators should be closed so that file garbage collection works.
|
|
Close() error
|
|
}
|