mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
zeekay
9a90943b8f
13,838 upstream commits, forward from the 26.6.1.1 fork point. 129 conflicts: 37 resolved mechanically (take upstream's file, re-apply the brand sweep) and 92 by hand where our fork and upstream had both changed the same code. The rule throughout: upstream owns the engine, we own the naming and a small set of deliberate deviations. Never resolve by discarding an upstream change — a dropped rename or a missed new parameter becomes a compile error a thousand files away. Deviations verified intact after the merge: ENABLE_XRAY still defaults OFF (CMakeLists.txt:105) X-Datastore-User read by the server (authenticateUserByHTTP.cpp) X-ClickHouse-* in src/ 0 hanzo/research present system.*_log retention bound present ArrowFlight rip held against upstream re-adding it Two things worth naming, because neither shows up as a conflict: src/CMakeLists.txt gained an UNGUARDED `add_object_library(datastore_storages_arrowflight Storages/ArrowFlight)` from upstream, for a directory our gRPC rip deleted. It sat outside every conflict hunk, so a resolver reading only the markers would sail straight past it into a CMake configure error. Removed; the Server/ArrowFlight sibling at line 474 stays because it is inside `if (TARGET ch_contrib::arrow_flight)`. The brand sweep had rewritten github.com/ClickHouse/ClickHouse issue links to github.com/Datastore/Datastore — a dead host. Those point at upstream's own tracker and are restored, alongside the contributors list, sql.clickhouse.com and the ClickHouse Inc copyright: attribution is theirs, not ours to rename. base/glibc-compatibility/memcpy/memcpy.h is deleted, not kept: upstream dropped it for LLVM-libc's, and our only change to it was a comment rebrand.
75 lines
3.3 KiB
C++
75 lines
3.3 KiB
C++
#include <iostream>
|
|
#include <boost/program_options.hpp>
|
|
|
|
#include <Coordination/CoordinationSettings.h>
|
|
#include <Coordination/KeeperSnapshotManager.h>
|
|
#include <Coordination/KeeperStorage.h>
|
|
#include <Coordination/KeeperStorageImpl.h>
|
|
#include <Coordination/KeeperMemNodesStorage.h>
|
|
#include <Coordination/KeeperStorage_fwd.h>
|
|
#include <Coordination/ZooKeeperDataReader.h>
|
|
#include <Coordination/KeeperContext.h>
|
|
#include <Common/TerminalSize.h>
|
|
#include <Poco/ConsoleChannel.h>
|
|
#include <Poco/AutoPtr.h>
|
|
#include <Poco/Logger.h>
|
|
#include <Common/logger_useful.h>
|
|
#include <Disks/DiskLocal.h>
|
|
|
|
|
|
int mainEntryDatastoreKeeperConverter(int argc, char ** argv);
|
|
int mainEntryDatastoreKeeperConverter(int argc, char ** argv)
|
|
{
|
|
using namespace DB;
|
|
namespace po = boost::program_options;
|
|
|
|
po::options_description desc = createOptionsDescription("Allowed options", getTerminalWidth());
|
|
desc.add_options()
|
|
("help,h", "produce help message")
|
|
("zookeeper-logs-dir", po::value<std::string>(), "Path to directory with ZooKeeper logs")
|
|
("zookeeper-snapshots-dir", po::value<std::string>(), "Path to directory with ZooKeeper snapshots")
|
|
("output-dir", po::value<std::string>(), "Directory to place output datastore-keeper snapshot")
|
|
;
|
|
po::variables_map options;
|
|
po::store(po::command_line_parser(argc, argv).options(desc).run(), options);
|
|
Poco::AutoPtr<Poco::ConsoleChannel> console_channel(new Poco::ConsoleChannel);
|
|
|
|
LoggerPtr logger = getLogger("KeeperConverter");
|
|
logger->setChannel(console_channel);
|
|
|
|
if (options.contains("help"))
|
|
{
|
|
std::cout << "Usage: datastore keeper-converter --zookeeper-logs-dir /var/lib/zookeeper/data/version-2 --zookeeper-snapshots-dir /var/lib/zookeeper/data/version-2 --output-dir /var/lib/datastore/coordination/snapshots" << std::endl;
|
|
std::cout << desc << std::endl;
|
|
return 0;
|
|
}
|
|
|
|
try
|
|
{
|
|
auto keeper_context = std::make_shared<KeeperContext>(true, std::make_shared<CoordinationSettings>());
|
|
keeper_context->setDigestEnabled(true);
|
|
keeper_context->setSnapshotDisk(std::make_shared<DiskLocal>("Keeper-snapshots", options["output-dir"].as<std::string>()));
|
|
|
|
DB::KeeperMemoryStorage storage(/* tick_time_ms */ 500, /* superdigest */ "", keeper_context);
|
|
|
|
DB::deserializeKeeperStorageFromSnapshotsDir(storage, options["zookeeper-snapshots-dir"].as<std::string>(), logger);
|
|
storage.initializeSystemNodes();
|
|
|
|
DB::deserializeLogsAndApplyToStorage(storage, options["zookeeper-logs-dir"].as<std::string>(), logger);
|
|
DB::SnapshotMetadataPtr snapshot_meta = std::make_shared<DB::SnapshotMetadata>(storage.getZXID(), 1, std::make_shared<nuraft::cluster_config>());
|
|
DB::KeeperStorageSnapshot snapshot(&storage, snapshot_meta, nullptr, keeper_context->getWriteSnapshotVersion());
|
|
|
|
DB::KeeperSnapshotManager manager(1, keeper_context);
|
|
auto snp = manager.serializeSnapshotToBuffer(snapshot);
|
|
auto file_info = manager.serializeSnapshotBufferToDisk(*snp, storage.getZXID());
|
|
std::cout << "Snapshot serialized to path:" << fs::path(file_info->disk->getPath()) / file_info->path << std::endl;
|
|
}
|
|
catch (...)
|
|
{
|
|
std::cerr << getCurrentExceptionMessage(true) << '\n';
|
|
return getCurrentExceptionCode();
|
|
}
|
|
|
|
return 0;
|
|
}
|