mirror of
https://github.com/luxfi/zapdb.git
synced 2026-10-30 15:12:20 +00:00
Manish R JainandGitHub
e2b6d47b29
This PR adds FullCopy option in Stream. This allows sending the table entirely to the writer. If this option is set to true we directly copy over the tables from the last 2 levels. This option increases the stream speed while also lowering the memory consumption on the DB that is streaming the KVs. For 71GB, compressed and encrypted DB we observed 3x improvement in speed. The DB contained ~65GB in the last 2 levels while remaining in the above levels. To use this option, the following options should be set in Stream. stream.KeyToList = nil stream.ChooseKey = nil stream.SinceTs = 0 db.managedTxns = true If we use stream writer for receiving the KVs, the encryption mode has to be the same in sender and receiver. This will restrict db.StreamDB() to use the same encryption mode in both input and output DB. Added TODO for allowing different encryption modes.
95 lines
2.1 KiB
Protocol Buffer
95 lines
2.1 KiB
Protocol Buffer
/*
|
|
* Copyright (C) 2017 Dgraph Labs, Inc. and Contributors
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
// Use protos/gen.sh to generate .pb.go files.
|
|
syntax = "proto3";
|
|
|
|
package badgerpb3;
|
|
|
|
option go_package = "github.com/dgraph-io/badger/v3/pb";
|
|
|
|
message KV {
|
|
bytes key = 1;
|
|
bytes value = 2;
|
|
bytes user_meta = 3;
|
|
uint64 version = 4;
|
|
uint64 expires_at = 5;
|
|
bytes meta = 6;
|
|
|
|
// Stream id is used to identify which stream the KV came from.
|
|
uint32 stream_id = 10;
|
|
// Stream done is used to indicate end of stream.
|
|
bool stream_done = 11;
|
|
|
|
enum Kind {
|
|
KEY = 0;
|
|
DATA_KEY = 1;
|
|
FILE = 2;
|
|
}
|
|
Kind kind = 12;
|
|
}
|
|
|
|
message KVList {
|
|
repeated KV kv = 1;
|
|
|
|
// alloc_ref used internally for memory management.
|
|
uint64 alloc_ref = 10;
|
|
}
|
|
|
|
message ManifestChangeSet {
|
|
// A set of changes that are applied atomically.
|
|
repeated ManifestChange changes = 1;
|
|
}
|
|
|
|
enum EncryptionAlgo {
|
|
aes = 0;
|
|
}
|
|
|
|
message ManifestChange {
|
|
uint64 Id = 1; // Table ID.
|
|
enum Operation {
|
|
CREATE = 0;
|
|
DELETE = 1;
|
|
}
|
|
Operation Op = 2;
|
|
uint32 Level = 3; // Only used for CREATE.
|
|
uint64 key_id = 4;
|
|
EncryptionAlgo encryption_algo = 5;
|
|
uint32 compression = 6; // Only used for CREATE Op.
|
|
}
|
|
|
|
message Checksum {
|
|
enum Algorithm {
|
|
CRC32C = 0;
|
|
XXHash64 = 1;
|
|
}
|
|
Algorithm algo = 1; // For storing type of Checksum algorithm used
|
|
uint64 sum = 2;
|
|
}
|
|
|
|
message DataKey {
|
|
uint64 key_id = 1;
|
|
bytes data = 2;
|
|
bytes iv = 3;
|
|
int64 created_at = 4;
|
|
}
|
|
|
|
message Match {
|
|
bytes prefix = 1;
|
|
string ignore_bytes = 2; // Comma separated with dash to represent ranges "1, 2-3, 4-7, 9"
|
|
}
|
|
|