#include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "config.h" namespace DB { namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int UNKNOWN_CODEC; extern const int UNEXPECTED_AST_STRUCTURE; extern const int DATA_TYPE_CANNOT_HAVE_ARGUMENTS; } CompressionCodecPtr CompressionCodecFactory::getDefaultCodec() const { return default_codec; } CompressionCodecPtr CompressionCodecFactory::get(const String & family_name, std::optional level) const { if (level) { auto level_literal = make_intrusive(static_cast(*level)); return get(makeASTFunction("CODEC", makeASTFunction(Poco::toUpper(family_name), level_literal)), {}); } auto identifier = make_intrusive(Poco::toUpper(family_name)); return get(makeASTFunction("CODEC", identifier), {}); } CompressionCodecPtr CompressionCodecFactory::get(const String & compression_codec) const { ParserCodec codec_parser; auto ast = parseQuery(codec_parser, "(" + Poco::toUpper(compression_codec) + ")", 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); return CompressionCodecFactory::instance().get(ast, nullptr); } CompressionCodecPtr CompressionCodecFactory::get( const ASTPtr & ast, const IDataType * column_type, CompressionCodecPtr current_default, bool only_generic) const { if (current_default == nullptr) current_default = default_codec; if (const auto * func = ast->as()) { Codecs codecs; codecs.reserve(func->arguments->children.size()); for (const auto & inner_codec_ast : func->arguments->children) { String codec_family_name; ASTPtr codec_arguments; if (const auto * family_name = inner_codec_ast->as()) { codec_family_name = family_name->name(); codec_arguments = {}; } else if (const auto * ast_func = inner_codec_ast->as()) { codec_family_name = ast_func->name; codec_arguments = ast_func->arguments; } else throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, "Unexpected AST element for compression codec"); CompressionCodecPtr codec; if (codec_family_name == DEFAULT_CODEC_NAME) codec = current_default; else codec = getImpl(codec_family_name, codec_arguments, column_type); if (only_generic && !codec->isGenericCompression()) continue; codecs.emplace_back(codec); } CompressionCodecPtr res; if (codecs.size() == 1) return codecs.back(); if (codecs.size() > 1) return std::make_shared(codecs); return std::make_shared(); } throw Exception(ErrorCodes::UNEXPECTED_AST_STRUCTURE, "Unexpected AST structure for compression codec: {}", ast->formatForErrorMessage()); } CompressionCodecPtr CompressionCodecFactory::get(uint8_t byte_code) const { const auto family_code_and_creator = family_code_with_codec.find(byte_code); if (family_code_and_creator == family_code_with_codec.end()) throw Exception(ErrorCodes::UNKNOWN_CODEC, "Unknown codec family code: {}", toString(byte_code)); return family_code_and_creator->second({}, nullptr); } void CompressionCodecFactory::fillCodecDescriptions(MutableColumns & res_columns) const { std::for_each( family_name_with_codec.begin(), family_name_with_codec.end(), [&](const auto &it) { const std::string &name = it.first; CompressionCodecPtr tmp = it.second({}, nullptr); res_columns[0]->insert(name); res_columns[1]->insert(tmp->getMethodByte()); res_columns[2]->insert(tmp->isCompression()); res_columns[3]->insert(tmp->isGenericCompression()); res_columns[4]->insert(tmp->isEncryption()); res_columns[5]->insert(tmp->isFloatingPointTimeSeriesCodec()); res_columns[6]->insert(tmp->isExperimental()); res_columns[7]->insert(tmp->getDescription()); } ); } std::vector> CompressionCodecFactory::getCodecDocumentations() const { std::vector> result; result.reserve(family_name_with_codec.size()); for (const auto & [name, creator] : family_name_with_codec) { CompressionCodecPtr codec = creator({}, nullptr); Documentation documentation; documentation.description = codec->getDescription(); /// The codec carries its description through `getDescription` rather than a `Documentation` object, so the /// source is not captured automatically; use the registration site recorded in `registerCompressionCodec*`. if (auto it = family_name_with_source.find(name); it != family_name_with_source.end()) documentation.source = it->second; result.emplace_back(name, std::move(documentation)); } return result; } CompressionCodecPtr CompressionCodecFactory::getImpl(const String & family_name, const ASTPtr & arguments, const IDataType * column_type) const { if (family_name == "Multiple") throw Exception(ErrorCodes::UNKNOWN_CODEC, "Codec Multiple cannot be specified directly"); const auto family_and_creator = family_name_with_codec.find(family_name); if (family_and_creator == family_name_with_codec.end()) throw Exception(ErrorCodes::UNKNOWN_CODEC, "Unknown codec family: {}", family_name); return family_and_creator->second(arguments, column_type); } void CompressionCodecFactory::registerCompressionCodecWithType( const String & family_name, std::optional byte_code, CreatorWithType creator, std::source_location source) { if (creator == nullptr) throw Exception(ErrorCodes::LOGICAL_ERROR, "CompressionCodecFactory: " "the codec family {} has been provided a null constructor", family_name); if (!family_name_with_codec.emplace(family_name, creator).second) throw Exception(ErrorCodes::LOGICAL_ERROR, "CompressionCodecFactory: the codec family name '{}' is not unique", family_name); family_name_with_source.emplace(family_name, source.file_name()); if (byte_code) if (!family_code_with_codec.emplace(*byte_code, creator).second) throw Exception(ErrorCodes::LOGICAL_ERROR, "CompressionCodecFactory: the codec family code '{}' is not unique", std::to_string(*byte_code)); } void CompressionCodecFactory::registerCompressionCodec(const String & family_name, std::optional byte_code, Creator creator, std::source_location source) { registerCompressionCodecWithType(family_name, byte_code, [family_name, creator](const ASTPtr & ast, const IDataType * /* data_type */) { return creator(ast); }, source); } void CompressionCodecFactory::registerSimpleCompressionCodec( const String & family_name, std::optional byte_code, SimpleCreator creator, std::source_location source) { registerCompressionCodec(family_name, byte_code, [family_name, creator](const ASTPtr & ast) { if (ast) throw Exception(ErrorCodes::DATA_TYPE_CANNOT_HAVE_ARGUMENTS, "Compression codec {} cannot have arguments", family_name); return creator(); }, source); } Strings CompressionCodecFactory::getAllRegisteredNames() const { Strings result; result.reserve(family_name_with_codec.size()); for (const auto & pair : family_name_with_codec) result.push_back(pair.first); return result; } /// Defined in individual CompressionCodec*.cpp files /// and declared in registerCompressionCodecs.h CompressionCodecFactory::CompressionCodecFactory() { registerCodecNone(*this); registerCodecLZ4(*this); registerCodecZSTD(*this); registerCodecLZ4HC(*this); registerCodecMultiple(*this); registerCodecDelta(*this); registerCodecT64(*this); registerCodecDoubleDelta(*this); registerCodecGorilla(*this); registerCodecEncrypted(*this); registerCodecFPC(*this); registerCodecGCD(*this); registerCodecALP(*this); default_codec = get("LZ4", {}); } CompressionCodecFactory & CompressionCodecFactory::instance() { static CompressionCodecFactory ret; return ret; } }