mirror of
https://github.com/hanzoai/datastore.git
synced 2026-09-26 13:55:07 +00:00
Alexey MilovidovandClaude Opus 4.8
0c48a0027e
Extend `system.documentation` with a `source` column holding the path of the source file where each entity's documentation is defined, relative to the repository root. The path is captured automatically at the construction site of every `Documentation`/`FunctionDocumentation` object via `std::source_location` (and normalized to be repository-relative regardless of build path mapping), and is provided explicitly for the entity groups documented in a single source file. Also document additional kinds of entities: compression codecs (`Compression Codec`), profile events (`Profile Event`), current metrics (`Current Metric`), asynchronous metrics (`Asynchronous Metric`), and the system tables themselves (`System Table`). For system tables the documentation is the table comment together with the list of its columns (name, type and description); for settings of any kind the documentation now also includes the type and default value. Co-Authored-By: Claude Opus 4.8 (1M context) <[email protected]>
256 lines
8.9 KiB
C++
256 lines
8.9 KiB
C++
#include <Compression/CompressionFactory.h>
|
|
#include <Compression/CompressionCodecMultiple.h>
|
|
#include <Compression/CompressionCodecNone.h>
|
|
#include <Compression/registerCompressionCodecs.h>
|
|
#include <IO/ReadBuffer.h>
|
|
#include <IO/WriteHelpers.h>
|
|
#include <Parsers/ASTFunction.h>
|
|
#include <Parsers/ASTIdentifier.h>
|
|
#include <Parsers/ASTLiteral.h>
|
|
#include <Parsers/ExpressionElementParsers.h>
|
|
#include <Parsers/parseQuery.h>
|
|
#include <Poco/String.h>
|
|
|
|
#include <Columns/IColumn.h>
|
|
#include <algorithm>
|
|
|
|
#include <boost/algorithm/string/join.hpp>
|
|
|
|
#include "config.h"
|
|
|
|
namespace DB
|
|
{
|
|
|
|
namespace ErrorCodes
|
|
{
|
|
extern const int LOGICAL_ERROR;
|
|
extern const int UNKNOWN_CODEC;
|
|
extern const int UNEXPECTED_AST_STRUCTURE;
|
|
extern const int DATA_TYPE_CANNOT_HAVE_ARGUMENTS;
|
|
}
|
|
|
|
CompressionCodecPtr CompressionCodecFactory::getDefaultCodec() const
|
|
{
|
|
return default_codec;
|
|
}
|
|
|
|
|
|
CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std::optional<int> level) const
|
|
{
|
|
if (level)
|
|
{
|
|
auto level_literal = make_intrusive<ASTLiteral>(static_cast<UInt64>(*level));
|
|
return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), level_literal)), {});
|
|
}
|
|
|
|
auto identifier = make_intrusive<ASTIdentifier>(Poco::toUpper(family_name));
|
|
return get(makeASTFunction("CODEC", identifier), {});
|
|
}
|
|
|
|
CompressionCodecPtr CompressionCodecFactory::get(const String & compression_codec) const
|
|
{
|
|
ParserCodec codec_parser;
|
|
auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS);
|
|
return CompressionCodecFactory::instance().get(ast, nullptr);
|
|
}
|
|
|
|
CompressionCodecPtr CompressionCodecFactory::get(
|
|
const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default, bool only_generic) const
|
|
{
|
|
if (current_default == nullptr)
|
|
current_default = default_codec;
|
|
|
|
if (const auto * func = ast->as<ASTFunction>())
|
|
{
|
|
Codecs codecs;
|
|
codecs.reserve(func->arguments->children.size());
|
|
for (const auto & inner_codec_ast : func->arguments->children)
|
|
{
|
|
String codec_family_name;
|
|
ASTPtr codec_arguments;
|
|
if (const auto * family_name = inner_codec_ast->as<ASTIdentifier>())
|
|
{
|
|
codec_family_name = family_name->name();
|
|
codec_arguments = {};
|
|
}
|
|
else if (const auto * ast_func = inner_codec_ast->as<ASTFunction>())
|
|
{
|
|
codec_family_name = ast_func->name;
|
|
codec_arguments = ast_func->arguments;
|
|
}
|
|
else
|
|
throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, "Unexpected AST element for compression codec");
|
|
|
|
CompressionCodecPtr codec;
|
|
if (codec_family_name == DEFAULT_CODEC_NAME)
|
|
codec = current_default;
|
|
else
|
|
codec = getImpl(codec_family_name, codec_arguments, column_type);
|
|
|
|
if (only_generic && !codec->isGenericCompression())
|
|
continue;
|
|
|
|
codecs.emplace_back(codec);
|
|
}
|
|
|
|
CompressionCodecPtr res;
|
|
|
|
if (codecs.size() == 1)
|
|
return codecs.back();
|
|
if (codecs.size() > 1)
|
|
return std::make_shared<CompressionCodecMultiple>(codecs);
|
|
return std::make_shared<CompressionCodecNone>();
|
|
}
|
|
|
|
throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, "Unexpected AST structure for compression codec: {}", ast->formatForErrorMessage());
|
|
}
|
|
|
|
|
|
CompressionCodecPtr CompressionCodecFactory::get(uint8_t byte_code) const
|
|
{
|
|
const auto family_code_and_creator = family_code_with_codec.find(byte_code);
|
|
|
|
if (family_code_and_creator == family_code_with_codec.end())
|
|
throw Exception(ErrorCodes::UNKNOWN_CODEC, "Unknown codec family code: {}", toString(byte_code));
|
|
|
|
return family_code_and_creator->second({}, nullptr);
|
|
}
|
|
|
|
void CompressionCodecFactory::fillCodecDescriptions(MutableColumns & res_columns) const
|
|
{
|
|
std::for_each(
|
|
family_name_with_codec.begin(),
|
|
family_name_with_codec.end(),
|
|
[&](const auto &it)
|
|
{
|
|
const std::string &name = it.first;
|
|
CompressionCodecPtr tmp = it.second({}, nullptr);
|
|
|
|
res_columns[0]->insert(name);
|
|
res_columns[1]->insert(tmp->getMethodByte());
|
|
res_columns[2]->insert(tmp->isCompression());
|
|
res_columns[3]->insert(tmp->isGenericCompression());
|
|
res_columns[4]->insert(tmp->isEncryption());
|
|
res_columns[5]->insert(tmp->isFloatingPointTimeSeriesCodec());
|
|
res_columns[6]->insert(tmp->isExperimental());
|
|
res_columns[7]->insert(tmp->getDescription());
|
|
}
|
|
);
|
|
}
|
|
|
|
std::vector<std::pair<String, Documentation>> CompressionCodecFactory::getCodecDocumentations() const
|
|
{
|
|
std::vector<std::pair<String, Documentation>> result;
|
|
result.reserve(family_name_with_codec.size());
|
|
for (const auto & [name, creator] : family_name_with_codec)
|
|
{
|
|
CompressionCodecPtr codec = creator({}, nullptr);
|
|
Documentation documentation;
|
|
documentation.description = codec->getDescription();
|
|
/// The codec carries its description through `getDescription` rather than a `Documentation` object, so the
|
|
/// source is not captured automatically; use the registration site recorded in `registerCompressionCodec*`.
|
|
if (auto it = family_name_with_source.find(name); it != family_name_with_source.end())
|
|
documentation.source = it->second;
|
|
result.emplace_back(name, std::move(documentation));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
CompressionCodecPtr CompressionCodecFactory::getImpl(const String & family_name, const ASTPtr & arguments, const IDataType * column_type) const
|
|
{
|
|
if (family_name == "Multiple")
|
|
throw Exception(ErrorCodes::UNKNOWN_CODEC, "Codec Multiple cannot be specified directly");
|
|
|
|
const auto family_and_creator = family_name_with_codec.find(family_name);
|
|
|
|
if (family_and_creator == family_name_with_codec.end())
|
|
throw Exception(ErrorCodes::UNKNOWN_CODEC, "Unknown codec family: {}", family_name);
|
|
|
|
return family_and_creator->second(arguments, column_type);
|
|
}
|
|
|
|
void CompressionCodecFactory::registerCompressionCodecWithType(
|
|
const String & family_name,
|
|
std::optional<uint8_t> byte_code,
|
|
CreatorWithType creator,
|
|
std::source_location source)
|
|
{
|
|
if (creator == nullptr)
|
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "CompressionCodecFactory: "
|
|
"the codec family {} has been provided a null constructor", family_name);
|
|
|
|
if (!family_name_with_codec.emplace(family_name, creator).second)
|
|
throw Exception(ErrorCodes::LOGICAL_ERROR, "CompressionCodecFactory: the codec family name '{}' is not unique", family_name);
|
|
|
|
family_name_with_source.emplace(family_name, source.file_name());
|
|
|
|
if (byte_code)
|
|
if (!family_code_with_codec.emplace(*byte_code, creator).second)
|
|
throw Exception(ErrorCodes::LOGICAL_ERROR,
|
|
"CompressionCodecFactory: the codec family code '{}' is not unique",
|
|
std::to_string(*byte_code));
|
|
}
|
|
|
|
void CompressionCodecFactory::registerCompressionCodec(const String & family_name, std::optional<uint8_t> byte_code, Creator creator, std::source_location source)
|
|
{
|
|
registerCompressionCodecWithType(family_name, byte_code, [family_name, creator](const ASTPtr & ast, const IDataType * /* data_type */)
|
|
{
|
|
return creator(ast);
|
|
}, source);
|
|
}
|
|
|
|
void CompressionCodecFactory::registerSimpleCompressionCodec(
|
|
const String & family_name,
|
|
std::optional<uint8_t> byte_code,
|
|
SimpleCreator creator,
|
|
std::source_location source)
|
|
{
|
|
registerCompressionCodec(family_name, byte_code, [family_name, creator](const ASTPtr & ast)
|
|
{
|
|
if (ast)
|
|
throw Exception(ErrorCodes::DATA_TYPE_CANNOT_HAVE_ARGUMENTS, "Compression codec {} cannot have arguments", family_name);
|
|
return creator();
|
|
}, source);
|
|
}
|
|
|
|
|
|
Strings CompressionCodecFactory::getAllRegisteredNames() const
|
|
{
|
|
Strings result;
|
|
result.reserve(family_name_with_codec.size());
|
|
for (const auto & pair : family_name_with_codec)
|
|
result.push_back(pair.first);
|
|
return result;
|
|
}
|
|
|
|
|
|
/// Defined in individual CompressionCodec*.cpp files
|
|
/// and declared in registerCompressionCodecs.h
|
|
|
|
CompressionCodecFactory::CompressionCodecFactory()
|
|
{
|
|
registerCodecNone(*this);
|
|
registerCodecLZ4(*this);
|
|
registerCodecZSTD(*this);
|
|
registerCodecLZ4HC(*this);
|
|
registerCodecMultiple(*this);
|
|
registerCodecDelta(*this);
|
|
registerCodecT64(*this);
|
|
registerCodecDoubleDelta(*this);
|
|
registerCodecGorilla(*this);
|
|
registerCodecEncrypted(*this);
|
|
registerCodecFPC(*this);
|
|
registerCodecGCD(*this);
|
|
registerCodecALP(*this);
|
|
|
|
default_codec = get("LZ4", {});
|
|
}
|
|
|
|
CompressionCodecFactory & CompressionCodecFactory::instance()
|
|
{
|
|
static CompressionCodecFactory ret;
|
|
return ret;
|
|
}
|
|
|
|
}
|