diff --git a/.mapping.json b/.mapping.json index ffd97b8dcad0..05b87f20d76f 100644 --- a/.mapping.json +++ b/.mapping.json @@ -1925,6 +1925,7 @@ "grpc/include/userver/ugrpc/client/impl/call_params.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/call_params.hpp", "grpc/include/userver/ugrpc/client/impl/channel_factory.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/channel_factory.hpp", "grpc/include/userver/ugrpc/client/impl/client_data.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/client_data.hpp", + "grpc/include/userver/ugrpc/client/impl/client_dependencies.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/client_dependencies.hpp", "grpc/include/userver/ugrpc/client/impl/codegen_declarations.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/codegen_declarations.hpp", "grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/codegen_definitions.hpp", "grpc/include/userver/ugrpc/client/impl/completion_queue_pool.hpp":"taxi/uservices/userver/grpc/include/userver/ugrpc/client/impl/completion_queue_pool.hpp", diff --git a/cmake/DetectVersion.cmake b/cmake/DetectVersion.cmake index d496501c723b..687028ddb041 100644 --- a/cmake/DetectVersion.cmake +++ b/cmake/DetectVersion.cmake @@ -95,9 +95,14 @@ function(pacman_version version_output_var pacmanpackage) ) if (version_result EQUAL 0) - if (version_output MATCHES "^(.*) (.*)-.*$") - set(${version_output_var} ${CMAKE_MATCH_2} PARENT_SCOPE) - message(STATUS "pacman version of ${pacmanpackage}: ${CMAKE_MATCH_2}") + # Possible output is `re2 1:20240702-2` (format spec https://wiki.archlinux.org/title/PKGBUILD#Version) + # `re2` is name + # `1` is epoch (optional) + # `20240702` is version + # `2` is release number + if (version_output MATCHES "^(.*) ([0-9]*:)?(.*)-.*$") + set(${version_output_var} ${CMAKE_MATCH_3} PARENT_SCOPE) + message(STATUS "pacman version of ${pacmanpackage}: ${CMAKE_MATCH_3}") else() set(${version_output_var} "NOT_FOUND") endif() diff --git a/core/include/userver/tracing/span.hpp b/core/include/userver/tracing/span.hpp index 1e51ad0bfd32..1f66dacf3d5e 100644 --- a/core/include/userver/tracing/span.hpp +++ b/core/include/userver/tracing/span.hpp @@ -23,7 +23,7 @@ class SpanBuilder; /// @brief Measures the execution time of the current code block, links it with /// the parent tracing::Spans and stores that info in the log. /// -/// Logging of spans can be controled at runtime via @ref USERVER_NO_LOG_SPANS. +/// Logging of spans can be controlled at runtime via @ref USERVER_NO_LOG_SPANS. /// /// See @ref scripts/docs/en/userver/logging.md for usage examples and more /// descriptions. diff --git a/core/include/userver/utils/statistics/labels.hpp b/core/include/userver/utils/statistics/labels.hpp index a9b5a4ccf294..b5073e2f2fd3 100644 --- a/core/include/userver/utils/statistics/labels.hpp +++ b/core/include/userver/utils/statistics/labels.hpp @@ -22,7 +22,7 @@ class LabelView final { LabelView(Label&& label) = delete; explicit LabelView(const Label& label); constexpr LabelView(std::string_view name, std::string_view value) : name_(name), value_(value) { - UINVARIANT(!name_.empty(), "The lable name must not be empty."); + UINVARIANT(!name_.empty(), "The label name must not be empty."); } template >* = nullptr> diff --git a/core/src/clients/http/client_crl_test.cpp b/core/src/clients/http/client_crl_test.cpp index 4da4af3c8278..2efaecbdc819 100644 --- a/core/src/clients/http/client_crl_test.cpp +++ b/core/src/clients/http/client_crl_test.cpp @@ -317,7 +317,7 @@ struct TlsServer { auto tls_server = engine::io::TlsWrapper::StartTlsServer( std::move(socket), - crypto::LoadCertficatesChainFromString(kServerCertificate), + crypto::LoadCertificatesChainFromString(kServerCertificate), crypto::PrivateKey::LoadFromString(kRevokedServerPrivateKey), deadline, cas diff --git a/core/src/engine/io/tls_wrapper_benchmark.cpp b/core/src/engine/io/tls_wrapper_benchmark.cpp index 193fe117fb47..7674f9a88454 100644 --- a/core/src/engine/io/tls_wrapper_benchmark.cpp +++ b/core/src/engine/io/tls_wrapper_benchmark.cpp @@ -97,7 +97,7 @@ constexpr auto kDeadlineMaxTime = std::chrono::seconds{60}; [&reading, deadline](auto&& server) { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), deadline ); @@ -141,7 +141,7 @@ BENCHMARK(tls_write_all_buffered)->RangeMultiplier(2)->Range(1 << 6, 1 << 12)->U [&reading, deadline](auto&& server) { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), deadline ); diff --git a/core/src/engine/io/tls_wrapper_test.cpp b/core/src/engine/io/tls_wrapper_test.cpp index fd0bf3d06a40..37a26d9b23ff 100644 --- a/core/src/engine/io/tls_wrapper_test.cpp +++ b/core/src/engine/io/tls_wrapper_test.cpp @@ -234,7 +234,7 @@ UTEST(TlsWrapper, InitListSmall) { [deadline, kDataA, kDataB, kDataC, kDataD](auto&& server) { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert_chain), + crypto::LoadCertificatesChainFromString(cert_chain), crypto::PrivateKey::LoadFromString(chain_private_key), deadline ); @@ -274,7 +274,7 @@ UTEST(TlsWrapper, InitListLarge) { [deadline, kDataA, kDataB, kDataC, kDataD](auto&& server) { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), deadline ); @@ -310,7 +310,7 @@ UTEST(TlsWrapper, InitListSmallThenLarge) { [deadline, kDataSmall, kDataLarge](auto&& server) { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), deadline ); @@ -342,7 +342,7 @@ UTEST_MT(TlsWrapper, Smoke, 2) { try { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), test_deadline ); @@ -389,7 +389,7 @@ UTEST_MT(TlsWrapper, DocTest, 2) { [deadline](auto&& server) { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), deadline ); @@ -421,7 +421,7 @@ UTEST(TlsWrapper, Move) { try { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), test_deadline ); @@ -471,7 +471,7 @@ UTEST(TlsWrapper, ConnectTimeout) { EXPECT_THROW( static_cast(io::TlsWrapper::StartTlsServer( std::move(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), Deadline::FromDuration(kShortTimeout) )), @@ -490,7 +490,7 @@ UTEST_MT(TlsWrapper, IoTimeout, 2) { [test_deadline, &timeout_happened](auto&& server) { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), test_deadline ); @@ -527,7 +527,7 @@ UTEST(TlsWrapper, Cancel) { [test_deadline](auto&& server) { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), test_deadline ); @@ -554,7 +554,7 @@ UTEST_MT(TlsWrapper, CertKeyMismatch, 2) { UEXPECT_THROW( static_cast(io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(other_key), test_deadline )), @@ -581,7 +581,7 @@ UTEST_MT(TlsWrapper, NonTlsClient, 2) { UEXPECT_THROW( static_cast(io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(other_key), test_deadline )), @@ -622,7 +622,7 @@ UTEST_MT(TlsWrapper, DoubleSmoke, 4) { [test_deadline](auto&& server) { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), test_deadline ); @@ -643,7 +643,7 @@ UTEST_MT(TlsWrapper, DoubleSmoke, 4) { [test_deadline](auto&& server) { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(other_cert), + crypto::LoadCertificatesChainFromString(other_cert), crypto::PrivateKey::LoadFromString(other_key), test_deadline ); @@ -696,7 +696,7 @@ UTEST(TlsWrapper, InvalidSocket) { UEXPECT_THROW(static_cast(io::TlsWrapper::StartTlsClient({}, {}, test_deadline)), io::TlsException); UEXPECT_THROW( static_cast(io::TlsWrapper::StartTlsServer( - {}, crypto::LoadCertficatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), test_deadline + {}, crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), test_deadline )), io::TlsException ); @@ -713,7 +713,7 @@ UTEST(TlsWrapper, PeerShutdown) { try { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), test_deadline ); @@ -755,7 +755,7 @@ UTEST(TlsWrapper, PeerDisconnect) { try { auto tls_server = io::TlsWrapper::StartTlsServer( std::forward(server), - crypto::LoadCertficatesChainFromString(cert), + crypto::LoadCertificatesChainFromString(cert), crypto::PrivateKey::LoadFromString(key), test_deadline ); diff --git a/core/src/server/net/listener_config.cpp b/core/src/server/net/listener_config.cpp index 4d2d3dadc478..59475aa6609a 100644 --- a/core/src/server/net/listener_config.cpp +++ b/core/src/server/net/listener_config.cpp @@ -35,7 +35,7 @@ PortConfig Parse(const yaml_config::YamlConfig& value, formats::parse::To #include +#include #include USERVER_NAMESPACE_BEGIN @@ -38,7 +39,7 @@ class ClientFactory final { public: /// @brief Make a client of the specified code-generated type. template - Client MakeClient(ClientSettings&& settings); + Client MakeClient(ClientSettings&& client_settings); /// @deprecated Use the overload taking @ref ClientSettings instead. /// @brief Make a client of the specified code-generated type. @@ -50,7 +51,7 @@ class ClientFactory final { /// @cond // For internal use only. ClientFactory( - ClientFactorySettings&& settings, + ClientFactorySettings&& client_factory_settings, engine::TaskProcessor& channel_task_processor, MiddlewareFactories mws, ugrpc::impl::CompletionQueuePoolBase& completion_queues, @@ -63,7 +64,7 @@ class ClientFactory final { private: impl::ClientDependencies MakeClientDependencies(ClientSettings&& settings); - ClientFactorySettings settings_; + ClientFactorySettings client_factory_settings_; engine::TaskProcessor& channel_task_processor_; MiddlewareFactories mws_; ugrpc::impl::CompletionQueuePoolBase& completion_queues_; diff --git a/grpc/include/userver/ugrpc/client/client_factory_component.hpp b/grpc/include/userver/ugrpc/client/client_factory_component.hpp index ca952854c001..58af23061480 100644 --- a/grpc/include/userver/ugrpc/client/client_factory_component.hpp +++ b/grpc/include/userver/ugrpc/client/client_factory_component.hpp @@ -42,7 +42,6 @@ namespace ugrpc::client { /// /// Name | Description | Default value /// ---- | ----------- | ------------- -/// task-processor | the task processor for blocking channel creation | - /// channel-args | a map of channel arguments, see gRPC Core docs | {} /// auth-type | authentication method, see above | - /// default-service-config | default service config, see above | - diff --git a/grpc/include/userver/ugrpc/client/client_factory_settings.hpp b/grpc/include/userver/ugrpc/client/client_factory_settings.hpp index 7f05222a301e..b4a86e669216 100644 --- a/grpc/include/userver/ugrpc/client/client_factory_settings.hpp +++ b/grpc/include/userver/ugrpc/client/client_factory_settings.hpp @@ -10,8 +10,6 @@ #include #include -#include - USERVER_NAMESPACE_BEGIN namespace ugrpc::client { @@ -29,10 +27,6 @@ struct ClientFactorySettings final { /// @see https://grpc.github.io/grpc/core/group__grpc__arg__keys.html grpc::ChannelArguments channel_args{}; - /// The logging level override for the internal grpcpp library. Must be either - /// `kDebug`, `kInfo` or `kError`. - logging::Level native_log_level{logging::Level::kError}; - /// Number of underlying channels that will be created for every client /// in this factory. std::size_t channel_count{1}; diff --git a/grpc/include/userver/ugrpc/client/impl/client_data.hpp b/grpc/include/userver/ugrpc/client/impl/client_data.hpp index f69569aa3f71..877544885571 100644 --- a/grpc/include/userver/ugrpc/client/impl/client_data.hpp +++ b/grpc/include/userver/ugrpc/client/impl/client_data.hpp @@ -7,16 +7,14 @@ #include -#include -#include +#include #include #include #include -#include -#include #include #include +#include #include #include #include @@ -25,28 +23,8 @@ USERVER_NAMESPACE_BEGIN -namespace ugrpc::impl { -class StatisticsStorage; -class CompletionQueuePoolBase; -} // namespace ugrpc::impl - namespace ugrpc::client::impl { -/// Contains all non-code-generated dependencies for creating a gRPC client -struct ClientDependencies final { - std::string client_name; - std::string endpoint; - Middlewares mws; - ugrpc::impl::CompletionQueuePoolBase& completion_queues; - ugrpc::impl::StatisticsStorage& statistics_storage; - dynamic_config::Source config_source; - testsuite::GrpcControl& testsuite_grpc; - const dynamic_config::Key* qos{nullptr}; - const ClientFactorySettings& client_factory_settings; - engine::TaskProcessor& channel_task_processor; - DedicatedMethodsConfig dedicated_methods_config; -}; - struct GenericClientTag final { explicit GenericClientTag() = default; }; diff --git a/grpc/include/userver/ugrpc/client/impl/client_dependencies.hpp b/grpc/include/userver/ugrpc/client/impl/client_dependencies.hpp new file mode 100644 index 000000000000..f5341dff3d90 --- /dev/null +++ b/grpc/include/userver/ugrpc/client/impl/client_dependencies.hpp @@ -0,0 +1,39 @@ +#pragma once + +#include + +#include +#include +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace ugrpc::impl { +class StatisticsStorage; +class CompletionQueuePoolBase; +} // namespace ugrpc::impl + +namespace ugrpc::client::impl { + +/// Contains all non-code-generated dependencies for creating a gRPC client +struct ClientDependencies final { + std::string client_name; + std::string endpoint; + Middlewares mws; + ugrpc::impl::CompletionQueuePoolBase& completion_queues; + ugrpc::impl::StatisticsStorage& statistics_storage; + dynamic_config::Source config_source; + testsuite::GrpcControl& testsuite_grpc; + const dynamic_config::Key* qos{nullptr}; + const ClientFactorySettings& client_factory_settings; + engine::TaskProcessor& channel_task_processor; + DedicatedMethodsConfig dedicated_methods_config; +}; + +} // namespace ugrpc::client::impl + +USERVER_NAMESPACE_END diff --git a/grpc/include/userver/ugrpc/client/impl/stub_pool.hpp b/grpc/include/userver/ugrpc/client/impl/stub_pool.hpp index b87cf1ebfafa..31526ca010bd 100644 --- a/grpc/include/userver/ugrpc/client/impl/stub_pool.hpp +++ b/grpc/include/userver/ugrpc/client/impl/stub_pool.hpp @@ -13,16 +13,6 @@ namespace ugrpc::client::impl { class StubPool final { public: - StubPool() = default; - - std::size_t Size() const { return stubs_.size(); } - - StubAny& NextStub() const; - - const utils::FixedArray>& GetChannels() const { return channels_; } - - const utils::FixedArray& GetStubs() const { return stubs_; } - template static StubPool Create(std::size_t size, const ChannelFactory& channel_factory) { auto channels = utils::GenerateFixedArray(size, [&channel_factory](std::size_t) { @@ -34,6 +24,16 @@ class StubPool final { return StubPool{std::move(channels), std::move(stubs)}; } + StubPool() = default; + + std::size_t Size() const { return stubs_.size(); } + + StubAny& NextStub() const; + + const utils::FixedArray>& GetChannels() const { return channels_; } + + const utils::FixedArray& GetStubs() const { return stubs_; } + private: StubPool(utils::FixedArray>&& channels, utils::FixedArray&& stubs) : channels_{std::move(channels)}, stubs_{std::move(stubs)} {} diff --git a/grpc/src/ugrpc/client/client_factory.cpp b/grpc/src/ugrpc/client/client_factory.cpp index 2593163decea..f693ed74613a 100644 --- a/grpc/src/ugrpc/client/client_factory.cpp +++ b/grpc/src/ugrpc/client/client_factory.cpp @@ -7,7 +7,7 @@ USERVER_NAMESPACE_BEGIN namespace ugrpc::client { ClientFactory::ClientFactory( - ClientFactorySettings&& settings, + ClientFactorySettings&& client_factory_settings, engine::TaskProcessor& channel_task_processor, MiddlewareFactories mws, ugrpc::impl::CompletionQueuePoolBase& completion_queues, @@ -15,33 +15,30 @@ ClientFactory::ClientFactory( testsuite::GrpcControl& testsuite_grpc, dynamic_config::Source config_source ) - : settings_(std::move(settings)), + : client_factory_settings_(std::move(client_factory_settings)), channel_task_processor_(channel_task_processor), mws_(mws), completion_queues_(completion_queues), client_statistics_storage_(statistics_storage), config_source_(config_source), - testsuite_grpc_(testsuite_grpc) { - ugrpc::impl::SetupNativeLogging(); - ugrpc::impl::UpdateNativeLogLevel(settings_.native_log_level); -} + testsuite_grpc_(testsuite_grpc) {} -impl::ClientDependencies ClientFactory::MakeClientDependencies(ClientSettings&& settings) { - UINVARIANT(!settings.client_name.empty(), "Client name is empty"); - UINVARIANT(!settings.endpoint.empty(), "Client endpoint is empty"); +impl::ClientDependencies ClientFactory::MakeClientDependencies(ClientSettings&& client_settings) { + UINVARIANT(!client_settings.client_name.empty(), "Client name is empty"); + UINVARIANT(!client_settings.endpoint.empty(), "Client endpoint is empty"); return impl::ClientDependencies{ - settings.client_name, - settings.endpoint, - impl::InstantiateMiddlewares(mws_, settings.client_name), + client_settings.client_name, + client_settings.endpoint, + impl::InstantiateMiddlewares(mws_, client_settings.client_name), completion_queues_, client_statistics_storage_, config_source_, testsuite_grpc_, - settings.client_qos, - settings_, + client_settings.client_qos, + client_factory_settings_, channel_task_processor_, - std::move(settings.dedicated_methods_config), + std::move(client_settings.dedicated_methods_config), }; } diff --git a/grpc/src/ugrpc/client/client_factory_component.cpp b/grpc/src/ugrpc/client/client_factory_component.cpp index debb2608e442..58b31b136148 100644 --- a/grpc/src/ugrpc/client/client_factory_component.cpp +++ b/grpc/src/ugrpc/client/client_factory_component.cpp @@ -40,7 +40,7 @@ ClientFactoryComponent::ClientFactoryComponent( const components::ComponentContext& context ) : ComponentBase(config, context) { - auto& processor_component = context.FindComponent(); + auto& client_common_component = context.FindComponent(); const auto config_source = context.FindComponent().GetSource(); @@ -54,10 +54,10 @@ ClientFactoryComponent::ClientFactoryComponent( factory_.emplace( MakeFactorySettings(std::move(factory_config), secdist), - processor_component.blocking_task_processor_, + client_common_component.blocking_task_processor_, std::move(middlewares), // - processor_component.completion_queues_, - processor_component.client_statistics_storage_, + client_common_component.completion_queues_, + client_common_component.client_statistics_storage_, testsuite_grpc, // config_source ); diff --git a/grpc/src/ugrpc/client/impl/client_factory_config.cpp b/grpc/src/ugrpc/client/impl/client_factory_config.cpp index 5a270b73ec20..b7888a6b5e28 100644 --- a/grpc/src/ugrpc/client/impl/client_factory_config.cpp +++ b/grpc/src/ugrpc/client/impl/client_factory_config.cpp @@ -64,24 +64,23 @@ ClientFactoryConfig Parse(const yaml_config::YamlConfig& value, formats::parse:: ClientFactorySettings MakeFactorySettings(ClientFactoryConfig&& config, const storages::secdist::SecdistConfig* secdist) { - auto creds = MakeDefaultCredentials(config.auth_type); - std::unordered_map> client_creds; + auto credentials = MakeDefaultCredentials(config.auth_type); + std::unordered_map> client_credentials; if (secdist) { const auto& tokens = secdist->Get(); for (const auto& [client_name, token] : tokens.tokens) { - client_creds[client_name] = grpc::CompositeChannelCredentials( - creds, grpc::AccessTokenCredentials(ugrpc::impl::ToGrpcString(token)) + client_credentials[client_name] = grpc::CompositeChannelCredentials( + credentials, grpc::AccessTokenCredentials(ugrpc::impl::ToGrpcString(token)) ); } } return ClientFactorySettings{ - creds, - client_creds, + credentials, + client_credentials, config.channel_args, - logging::Level::kError, config.channel_count, }; } diff --git a/grpc/src/ugrpc/client/impl/client_factory_config.hpp b/grpc/src/ugrpc/client/impl/client_factory_config.hpp index 3109488a10f5..c989b678e40b 100644 --- a/grpc/src/ugrpc/client/impl/client_factory_config.hpp +++ b/grpc/src/ugrpc/client/impl/client_factory_config.hpp @@ -3,7 +3,7 @@ #include #include -#include +#include USERVER_NAMESPACE_BEGIN diff --git a/redis/src/storages/redis/impl/cluster_sentinel_impl.cpp b/redis/src/storages/redis/impl/cluster_sentinel_impl.cpp index b27bb3a12cb7..09b5b54ed8d9 100644 --- a/redis/src/storages/redis/impl/cluster_sentinel_impl.cpp +++ b/redis/src/storages/redis/impl/cluster_sentinel_impl.cpp @@ -97,11 +97,11 @@ struct CommandSpecialPrinter { logging::LogHelper& operator<<(logging::LogHelper& os, CommandSpecialPrinter v) { const auto& command = v.command; - if (command->args.args.size() == 1 || command->invoke_counter + 1 >= command->args.args.size()) { + if (command->args.GetCommandCount() == 1 || command->invoke_counter + 1 >= command->args.GetCommandCount()) { os << command->args; - } else if (command->invoke_counter < command->args.args.size() && !command->args.args[command->invoke_counter].empty()) { + } else if (command->invoke_counter < command->args.GetCommandCount()) { os << fmt::format( - "subrequest idx={}, cmd={}", command->invoke_counter, command->args.args[command->invoke_counter].front() + "subrequest idx={}, cmd={}", command->invoke_counter, command->args.GetCommandName(command->invoke_counter) ); } @@ -614,9 +614,9 @@ void ClusterSentinelImpl::ProcessWaitingCommands() { const auto& command = scommand.command; const CommandControlImpl cc{command->control}; if (scommand.start + cc.timeout_all < now) { - for (const auto& args : command->args.args) { + for (const auto& args : command->args) { auto reply = std::make_shared( - args[0], nullptr, ReplyStatus::kTimeoutError, "Command in the send queue timed out" + args.GetCommandName(), nullptr, ReplyStatus::kTimeoutError, "Command in the send queue timed out" ); statistics_internal_.redis_not_ready++; InvokeCommand(command, std::move(reply)); @@ -637,9 +637,12 @@ void ClusterSentinelImpl::ProcessWaitingCommandsOnStop() { for (const SentinelCommand& scommand : waiting_commands) { const auto& command = scommand.command; - for (const auto& args : command->args.args) { + for (const auto& args : command->args) { auto reply = std::make_shared( - args[0], nullptr, ReplyStatus::kTimeoutError, "Stopping, killing commands remaining in send queue" + args.GetCommandName(), + nullptr, + ReplyStatus::kTimeoutError, + "Stopping, killing commands remaining in send queue" ); statistics_internal_.redis_not_ready++; InvokeCommand(command, std::move(reply)); @@ -892,10 +895,9 @@ void ClusterSentinelImpl::AsyncCommand(const SentinelCommand& scommand, size_t p const bool error_ask = reply->data.IsErrorAsk(); const bool error_moved = reply->data.IsErrorMoved(); if (error_moved) { - const auto& args = ccommand->args.args; LOG_DEBUG() << "MOVED" << reply->status_string << " c.instance_idx:" << ccommand->instance_idx << " shard: " << shard << " movedto:" << ParseMovedShard(reply->data.GetError()) - << " args:" << args; + << " args:" << ccommand->args; this->topology_holder_->SendUpdateClusterTopology(); } const bool retry_to_master = diff --git a/redis/src/storages/redis/impl/cluster_subscription_storage_test.cpp b/redis/src/storages/redis/impl/cluster_subscription_storage_test.cpp index ecbf2ed77945..f8ed02c1ffb6 100644 --- a/redis/src/storages/redis/impl/cluster_subscription_storage_test.cpp +++ b/redis/src/storages/redis/impl/cluster_subscription_storage_test.cpp @@ -73,8 +73,7 @@ class SubscriptionTest : public ::testing::Test { } void ProcessCommands() { for (auto& cmd : cmds_) { - const auto& command = cmd->args.args[0][0]; - const auto& channel = cmd->args.args[0][1]; + const auto& [command, channel] = cmd->args.GetCommandAndChannel(); storages::redis::ReplyData reply_data(storages::redis::ReplyData::Array{ storages::redis::ReplyData(command), storages::redis::ReplyData(channel), storages::redis::ReplyData(1)} ); diff --git a/redis/src/storages/redis/impl/cmd_args.cpp b/redis/src/storages/redis/impl/cmd_args.cpp index 1a36115cc915..e72ab5d2471c 100644 --- a/redis/src/storages/redis/impl/cmd_args.cpp +++ b/redis/src/storages/redis/impl/cmd_args.cpp @@ -9,51 +9,59 @@ USERVER_NAMESPACE_BEGIN namespace storages::redis::impl { -void PutArg(CmdArgs::CmdArgsArray& args_, const char* arg) { args_.emplace_back(arg); } +void CmdWithArgs::PutArg(const char* arg) { + UASSERT(arg); + UASSERT(arg[0] != '\0'); + args_.emplace_back(arg); +} + +void CmdWithArgs::PutArg(std::string_view arg) { args_.emplace_back(arg); } -void PutArg(CmdArgs::CmdArgsArray& args_, const std::string& arg) { args_.emplace_back(arg); } +void CmdWithArgs::PutArg(const std::string& arg) { args_.emplace_back(arg); } -void PutArg(CmdArgs::CmdArgsArray& args_, std::string&& arg) { args_.emplace_back(std::move(arg)); } +void CmdWithArgs::PutArg(std::string&& arg) { args_.emplace_back(std::move(arg)); } -void PutArg(CmdArgs::CmdArgsArray& args_, const std::vector& arg) { - for (const auto& str : arg) args_.emplace_back(str); +void CmdWithArgs::PutArg(const std::vector& arg) { + for (const auto& str : arg) { + args_.emplace_back(str); + } } -void PutArg(CmdArgs::CmdArgsArray& args_, const std::vector>& arg) { +void CmdWithArgs::PutArg(const std::vector>& arg) { for (const auto& pair : arg) { args_.emplace_back(pair.first); args_.emplace_back(pair.second); } } -void PutArg(CmdArgs::CmdArgsArray& args_, const std::vector>& arg) { +void CmdWithArgs::PutArg(const std::vector>& arg) { for (const auto& pair : arg) { args_.emplace_back(std::to_string(pair.first)); args_.emplace_back(pair.second); } } -void PutArg(CmdArgs::CmdArgsArray& args_, std::optional arg) { +void CmdWithArgs::PutArg(std::optional arg) { if (arg) { args_.emplace_back("MATCH"); args_.emplace_back(std::move(arg)->Get()); } } -void PutArg(CmdArgs::CmdArgsArray& args_, std::optional arg) { +void CmdWithArgs::PutArg(std::optional arg) { if (arg) { args_.emplace_back("COUNT"); args_.emplace_back(std::to_string(arg->Get())); } } -void PutArg(CmdArgs::CmdArgsArray& args_, GeoaddArg arg) { +void CmdWithArgs::PutArg(GeoaddArg arg) { args_.emplace_back(std::to_string(arg.lon)); args_.emplace_back(std::to_string(arg.lat)); args_.emplace_back(std::move(arg.member)); } -void PutArg(CmdArgs::CmdArgsArray& args_, std::vector arg) { +void CmdWithArgs::PutArg(std::vector arg) { for (auto& geoadd_arg : arg) { args_.emplace_back(std::to_string(geoadd_arg.lon)); args_.emplace_back(std::to_string(geoadd_arg.lat)); @@ -61,7 +69,7 @@ void PutArg(CmdArgs::CmdArgsArray& args_, std::vector arg) { } } -void PutArg(CmdArgs::CmdArgsArray& args_, const GeoradiusOptions& arg) { +void CmdWithArgs::PutArg(const GeoradiusOptions& arg) { switch (arg.unit) { case GeoradiusOptions::Unit::kM: args_.emplace_back("m"); @@ -90,7 +98,7 @@ void PutArg(CmdArgs::CmdArgsArray& args_, const GeoradiusOptions& arg) { args_.emplace_back("DESC"); } -void PutArg(CmdArgs::CmdArgsArray& args_, const GeosearchOptions& arg) { +void CmdWithArgs::PutArg(const GeosearchOptions& arg) { switch (arg.unit) { case GeosearchOptions::Unit::kM: args_.emplace_back("m"); @@ -119,7 +127,7 @@ void PutArg(CmdArgs::CmdArgsArray& args_, const GeosearchOptions& arg) { args_.emplace_back("DESC"); } -void PutArg(CmdArgs::CmdArgsArray& args_, const SetOptions& arg) { +void CmdWithArgs::PutArg(const SetOptions& arg) { if (arg.milliseconds) { args_.emplace_back("PX"); args_.emplace_back(std::to_string(arg.milliseconds)); @@ -133,7 +141,7 @@ void PutArg(CmdArgs::CmdArgsArray& args_, const SetOptions& arg) { args_.emplace_back("XX"); } -void PutArg(CmdArgs::CmdArgsArray& args_, const ZaddOptions& arg) { +void CmdWithArgs::PutArg(const ZaddOptions& arg) { if (arg.exist == ZaddOptions::Exist::kAddIfNotExist) args_.emplace_back("NX"); else if (arg.exist == ZaddOptions::Exist::kAddIfExist) @@ -147,11 +155,11 @@ void PutArg(CmdArgs::CmdArgsArray& args_, const ZaddOptions& arg) { if (arg.return_value == ZaddOptions::ReturnValue::kChangedCount) args_.emplace_back("CH"); } -void PutArg(CmdArgs::CmdArgsArray& args_, const ScoreOptions& arg) { +void CmdWithArgs::PutArg(const ScoreOptions& arg) { if (arg.withscores) args_.emplace_back("WITHSCORES"); } -void PutArg(CmdArgs::CmdArgsArray& args_, const RangeOptions& arg) { +void CmdWithArgs::PutArg(const RangeOptions& arg) { if (arg.offset || arg.count) { args_.emplace_back("LIMIT"); args_.emplace_back(arg.offset ? std::to_string(*arg.offset) : "0"); @@ -159,17 +167,47 @@ void PutArg(CmdArgs::CmdArgsArray& args_, const RangeOptions& arg) { } } -void PutArg(CmdArgs::CmdArgsArray& args_, const RangeScoreOptions& arg) { - PutArg(args_, arg.score_options); - PutArg(args_, arg.range_options); +void CmdWithArgs::PutArg(const RangeScoreOptions& arg) { + PutArg(arg.score_options); + PutArg(arg.range_options); } -logging::LogHelper& operator<<(logging::LogHelper& os, const CmdArgs& v) { +logging::LogHelper& operator<<(logging::LogHelper& os, const CmdWithArgs& v) { constexpr std::size_t kArgSizeLimit = 1024; - if (v.args.size() > 1) os << "["; + bool first_arg = true; + for (const auto& arg : v.args_) { + if (first_arg) + first_arg = false; + else { + os << " "; + if (os.IsLimitReached()) { + os << "..."; + break; + } + } + + if (utils::text::IsUtf8(arg)) { + if (arg.size() <= kArgSizeLimit) { + os << arg; + } else { + std::string_view view{arg}; + view = view.substr(0, kArgSizeLimit); + utils::text::utf8::TrimViewTruncatedEnding(view); + os << view << "<...>"; + } + } else { + os << ""; + } + } + + return os; +} + +logging::LogHelper& operator<<(logging::LogHelper& os, const CmdArgs& v) { + if (v.commands_.size() > 1) os << "["; bool first = true; - for (const auto& arg_array : v.args) { + for (const auto& arg_array : v.commands_) { if (first) first = false; else @@ -180,35 +218,9 @@ logging::LogHelper& operator<<(logging::LogHelper& os, const CmdArgs& v) { break; } - os << "\""; - bool first_arg = true; - for (const auto& arg : arg_array) { - if (first_arg) - first_arg = false; - else { - os << " "; - if (os.IsLimitReached()) { - os << "..."; - break; - } - } - - if (utils::text::IsUtf8(arg)) { - if (arg.size() <= kArgSizeLimit) { - os << arg; - } else { - std::string_view view{arg}; - view = view.substr(0, kArgSizeLimit); - utils::text::utf8::TrimViewTruncatedEnding(view); - os << view << "<...>"; - } - } else { - os << ""; - } - } - os << "\""; + os << "\"" << arg_array << "\""; } - if (v.args.size() > 1) os << "]"; + if (v.commands_.size() > 1) os << "]"; return os; } diff --git a/redis/src/storages/redis/impl/cmd_args.hpp b/redis/src/storages/redis/impl/cmd_args.hpp index 31e22eb62843..09b8bb4fd897 100644 --- a/redis/src/storages/redis/impl/cmd_args.hpp +++ b/redis/src/storages/redis/impl/cmd_args.hpp @@ -4,7 +4,12 @@ #include #include +#include +#include + #include +#include +#include #include #include @@ -13,77 +18,201 @@ USERVER_NAMESPACE_BEGIN namespace storages::redis::impl { -class CmdArgs { +class CmdArgs; + +class CmdWithArgs { + friend class CmdArgs; + public: - using CmdArgsArray = std::vector; - using CmdArgsChain = std::vector; + // channel is used for periodic subscribe/unsubscribe to calculate actual RTT + // instead of sending PING commands which are not supported by hiredis in + // subscriber mode + static constexpr std::string_view kSubscriberPingChannelName = "_ping_dummy_ch"; - CmdArgs() = default; + CmdWithArgs() = delete; + CmdWithArgs(const CmdWithArgs&) = default; + CmdWithArgs(CmdWithArgs&&) = default; + CmdWithArgs& operator=(const CmdWithArgs&) = default; + CmdWithArgs& operator=(CmdWithArgs&&) = default; template - CmdArgs(Args&&... _args) { - Then(std::forward(_args)...); + CmdWithArgs(std::string&& command_name, Args&&... arguments) { + UASSERT(!command_name.empty()); + UASSERT(command_name.size() == std::strlen(command_name.c_str())); + args_.reserve(1 + sizeof...(Args)); + PutArg(std::move(command_name)); + (this->PutArg(std::forward(arguments)), ...); } - CmdArgs(const CmdArgs& o) = delete; - CmdArgs(CmdArgs&& o) = default; + const std::string& GetCommandName() const noexcept { + UASSERT(!args_.empty()); + UASSERT(!args_[0].empty()); + return args_[0]; + } - CmdArgs& operator=(const CmdArgs& o) = delete; - CmdArgs& operator=(CmdArgs&& o) = default; + std::size_t GetCommandBytesLength() const { + std::size_t size = 0; + for (const auto& arg : args_) { + size += arg.size(); + } + UASSERT(size != 0); + return size; + } - template - CmdArgs& Then(Args&&... _args); + struct CommandChannel { + const std::string& command; + const std::string& channel; + }; - CmdArgs Clone() const { - CmdArgs r; - r.args = args; - return r; + CommandChannel GetCommandAndChannel() const noexcept { + UASSERT(args_.size() == 2); + return CommandChannel{args_[0], args_[1]}; + } + + bool IsMultiCommand() const noexcept { + static constexpr std::string_view multi_command{"MULTI"}; + return utils::StrIcaseEqual{}(GetCommandName(), multi_command); + } + + bool IsUnsubscribeCommand() const noexcept { + static constexpr std::string_view unsubscribe_command{"UNSUBSCRIBE"}; + static constexpr std::string_view punsubscribe_command{"PUNSUBSCRIBE"}; + static constexpr std::string_view sunsubscribe_command{"SUNSUBSCRIBE"}; + + return utils::StrIcaseEqual{}(GetCommandName(), unsubscribe_command) || + utils::StrIcaseEqual{}(GetCommandName(), punsubscribe_command) || + utils::StrIcaseEqual{}(GetCommandName(), sunsubscribe_command); + } + + bool IsSubscribeCommand() const noexcept { + static constexpr std::string_view subscribe_command{"SUBSCRIBE"}; + static constexpr std::string_view psubscribe_command{"PSUBSCRIBE"}; + static constexpr std::string_view ssubscribe_command{"SSUBSCRIBE"}; + + return utils::StrIcaseEqual{}(GetCommandName(), subscribe_command) || + utils::StrIcaseEqual{}(GetCommandName(), psubscribe_command) || + utils::StrIcaseEqual{}(GetCommandName(), ssubscribe_command); } - CmdArgsChain args; + bool IsSubscribesCommand() const noexcept { return IsSubscribeCommand() || IsUnsubscribeCommand(); } + + bool IsExecCommand() const noexcept { + static constexpr std::string_view exec_command{"EXEC"}; + return utils::StrIcaseEqual{}(GetCommandName(), exec_command); + } + + bool IsSubscriberPingChannel() const noexcept { return args_.size() > 1 && args_[1] == kSubscriberPingChannelName; } + + template + void FillPointerSizesStorages(PointersStorage& pointers, SizesStorage& sizes) const { + const auto size = args_.size(); + + pointers.reserve(size); + sizes.reserve(size); + + for (const auto& arg : args_) { + pointers.push_back(arg.data()); + sizes.push_back(arg.size()); + } + } + + auto GetJoinedArgs(const char* separator) const { return fmt::join(args_, separator); } + +private: + void PutArg(const char* arg); + + void PutArg(std::string_view arg); + + void PutArg(const std::string& arg); + + void PutArg(std::string&& arg); + + void PutArg(const std::vector& arg); + + void PutArg(const std::vector>& arg); + + void PutArg(const std::vector>& arg); + + void PutArg(std::optional arg); + void PutArg(std::optional arg); + void PutArg(GeoaddArg arg); + void PutArg(std::vector arg); + void PutArg(const GeoradiusOptions& arg); + void PutArg(const GeosearchOptions& arg); + void PutArg(const SetOptions& arg); + void PutArg(const ZaddOptions& arg); + void PutArg(const ScanOptions& arg); + + void PutArg(const ScoreOptions& arg); + void PutArg(const RangeOptions& arg); + void PutArg(const RangeScoreOptions& arg); + + template + typename std::enable_if::value, void>::type PutArg(const Arg& arg) { + args_.emplace_back(std::to_string(arg)); + } + + static constexpr std::size_t kTypicalArgsCount = 4; + boost::container::small_vector args_; + + friend logging::LogHelper& operator<<(logging::LogHelper& os, const CmdWithArgs& v); }; -void PutArg(CmdArgs::CmdArgsArray& args_, const char* arg); +class CmdArgs { +public: + template + CmdArgs(std::string command_name, Args&&... arguments) { + commands_.emplace_back(std::move(command_name), std::forward(arguments)...); + } -void PutArg(CmdArgs::CmdArgsArray& args_, const std::string& arg); + CmdArgs(const CmdArgs&) = delete; + CmdArgs(CmdArgs&&) = default; -void PutArg(CmdArgs::CmdArgsArray& args_, std::string&& arg); + CmdArgs& operator=(const CmdArgs&) = delete; + CmdArgs& operator=(CmdArgs&&) = default; -void PutArg(CmdArgs::CmdArgsArray& args_, const std::vector& arg); + template + CmdArgs& Then(std::string command_name, Args&&... arguments) { + commands_.emplace_back(std::move(command_name), std::forward(arguments)...); + return *this; + } + + CmdArgs Clone() const { + CmdArgs r; + r.commands_ = commands_; + return r; + } + + std::size_t GetCommandCount() const noexcept { return commands_.size(); } -void PutArg(CmdArgs::CmdArgsArray& args_, const std::vector>& arg); + const std::string& GetCommandName(std::size_t index) const noexcept { + UASSERT(commands_.size() > index); + return commands_[index].GetCommandName(); + } -void PutArg(CmdArgs::CmdArgsArray& args_, const std::vector>& arg); + auto GetCommandAndChannel() const noexcept { + UASSERT(commands_.size() == 1); + return commands_[0].GetCommandAndChannel(); + } -void PutArg(CmdArgs::CmdArgsArray& args_, std::optional arg); -void PutArg(CmdArgs::CmdArgsArray& args_, std::optional arg); -void PutArg(CmdArgs::CmdArgsArray& args_, GeoaddArg arg); -void PutArg(CmdArgs::CmdArgsArray& args_, std::vector arg); -void PutArg(CmdArgs::CmdArgsArray& args_, const GeoradiusOptions& arg); -void PutArg(CmdArgs::CmdArgsArray& args_, const GeosearchOptions& arg); -void PutArg(CmdArgs::CmdArgsArray& args_, const SetOptions& arg); -void PutArg(CmdArgs::CmdArgsArray& args_, const ZaddOptions& arg); -void PutArg(CmdArgs::CmdArgsArray& args_, const ScanOptions& arg); -void PutArg(CmdArgs::CmdArgsArray& args_, const ScoreOptions& arg); -void PutArg(CmdArgs::CmdArgsArray& args_, const RangeOptions& arg); -void PutArg(CmdArgs::CmdArgsArray& args_, const RangeScoreOptions& arg); + auto begin() const noexcept { + UASSERT(!commands_.empty()); + return commands_.cbegin(); + } -logging::LogHelper& operator<<(logging::LogHelper& os, const CmdArgs& v); + auto end() const noexcept { + UASSERT(!commands_.empty()); + return commands_.cend(); + } -template -typename std::enable_if::value, void>::type -PutArg(CmdArgs::CmdArgsArray& args_, const Arg& arg) { - args_.emplace_back(std::to_string(arg)); -} +private: + CmdArgs() = default; + + static constexpr std::size_t kTypicalCommandsCount = 1; + boost::container::small_vector commands_; -template -CmdArgs& CmdArgs::Then(Args&&... _args) { - args.emplace_back(); - auto& new_args = args.back(); - new_args.reserve(sizeof...(Args)); - (storages::redis::impl::PutArg(new_args, std::forward(_args)), ...); - return *this; -} + friend logging::LogHelper& operator<<(logging::LogHelper& os, const CmdArgs& v); +}; } // namespace storages::redis::impl diff --git a/redis/src/storages/redis/impl/command.cpp b/redis/src/storages/redis/impl/command.cpp index 98dc31d0f51a..c717ac0e46d1 100644 --- a/redis/src/storages/redis/impl/command.cpp +++ b/redis/src/storages/redis/impl/command.cpp @@ -10,6 +10,16 @@ USERVER_NAMESPACE_BEGIN namespace storages::redis::impl { +namespace { +std::string ToLower(std::string_view str) { + std::string result; + result.resize(str.size()); + std::transform(str.begin(), str.end(), result.begin(), [](unsigned char c) { return std::tolower(c); }); + return result; +} + +} // namespace + Command::Command( CmdArgs&& _args, ReplyCallback callback, @@ -28,12 +38,8 @@ Command::Command( counter(counter), asking(asking), redirected(redirected), - read_only(read_only) { - UASSERT_MSG(!args.args.empty() && !args.args.front().empty(), "Empty command make no sense"); - if (!args.args.empty() && !args.args.front().empty()) { - name = args.args.front().front(); - std::transform(name.begin(), name.end(), name.begin(), [](unsigned char c) { return std::tolower(c); }); - } + read_only(read_only), + name(ToLower(args.GetCommandName(0))) { if constexpr (utils::impl::kEnableAssert) { original_span_debug = tracing::Span::CurrentSpanUnchecked(); } diff --git a/redis/src/storages/redis/impl/command.hpp b/redis/src/storages/redis/impl/command.hpp index 2c07497f60b7..9efe6b0710c1 100644 --- a/redis/src/storages/redis/impl/command.hpp +++ b/redis/src/storages/redis/impl/command.hpp @@ -45,10 +45,10 @@ struct Command { size_t instance_idx = 0; uint32_t invoke_counter = 0; int counter = 0; - bool asking = false; - bool redirected = false; - bool read_only = false; - std::string name; + const bool asking = false; + const bool redirected = false; + const bool read_only = false; + const std::string name; }; CommandPtr PrepareCommand( diff --git a/redis/src/storages/redis/impl/redis.cpp b/redis/src/storages/redis/impl/redis.cpp index 27db27f975df..fcf748a6d5de 100644 --- a/redis/src/storages/redis/impl/redis.cpp +++ b/redis/src/storages/redis/impl/redis.cpp @@ -42,11 +42,6 @@ const auto kPingLatencyExp = 0.7; const auto kInitialPingLatencyMs = 1000; const size_t kMissedPingStreakThresholdDefault = 3; -// channel is used for periodic subscribe/unsubscribe to calculate actual RTT -// instead of sending PING commands which are not supported by hiredis in -// subscriber mode -const std::string kSubscriberPingChannelName = "_ping_dummy_ch"; - // required for libhiredis < 1.0.0 #ifndef REDIS_ERR_TIMEOUT // NOLINTNEXTLINE(cppcoreguidelines-macro-usage) @@ -73,46 +68,6 @@ ReplyStatus NativeToReplyStatus(int status) { return *reply_status; } -inline bool AreStringsEqualIgnoreCase(const std::string& l, const std::string& r) { - return l.size() == r.size() && !strcasecmp(l.c_str(), r.c_str()); -} - -inline bool IsUnsubscribeCommand(const CmdArgs::CmdArgsArray& args) { - static const std::string unsubscribe_command{"UNSUBSCRIBE"}; - static const std::string punsubscribe_command{"PUNSUBSCRIBE"}; - static const std::string sunsubscribe_command{"SUNSUBSCRIBE"}; - - return AreStringsEqualIgnoreCase(args[0], unsubscribe_command) || - AreStringsEqualIgnoreCase(args[0], punsubscribe_command) || - AreStringsEqualIgnoreCase(args[0], sunsubscribe_command); -} - -inline bool IsSubscribeCommand(const CmdArgs::CmdArgsArray& args) { - static const std::string subscribe_command{"SUBSCRIBE"}; - static const std::string psubscribe_command{"PSUBSCRIBE"}; - static const std::string ssubscribe_command{"SSUBSCRIBE"}; - - return AreStringsEqualIgnoreCase(args[0], subscribe_command) || - AreStringsEqualIgnoreCase(args[0], psubscribe_command) || - AreStringsEqualIgnoreCase(args[0], ssubscribe_command); -} - -inline bool IsSubscribesCommand(const CmdArgs::CmdArgsArray& args) { - return IsSubscribeCommand(args) || IsUnsubscribeCommand(args); -} - -inline bool IsMultiCommand(const CmdArgs::CmdArgsArray& args) { - static const std::string multi_command{"MULTI"}; - - return AreStringsEqualIgnoreCase(args[0], multi_command); -} - -inline bool IsExecCommand(const CmdArgs::CmdArgsArray& args) { - static const std::string exec_command{"EXEC"}; - - return AreStringsEqualIgnoreCase(args[0], exec_command); -} - bool IsFinalState(Redis::State state) { return state == Redis::State::kDisconnected || state == Redis::State::kDisconnectError; } @@ -698,7 +653,7 @@ void Redis::RedisImpl::SendSubscriberPing() { is_ping_in_flight_ = true; ProcessCommand(PrepareCommand( - CmdArgs{"SUBSCRIBE", kSubscriberPingChannelName}, + CmdArgs{"SUBSCRIBE", CmdWithArgs::kSubscriberPingChannelName}, [this](const CommandPtr&, ReplyPtr reply) { if (!*reply || !reply->data.IsArray()) { Disconnect(); @@ -710,7 +665,9 @@ void Redis::RedisImpl::SendSubscriberPing() { return; } if (!strcasecmp(reply_array[0].GetString().c_str(), "SUBSCRIBE")) { - ProcessCommand(PrepareCommand(CmdArgs{"UNSUBSCRIBE", kSubscriberPingChannelName}, ReplyCallback{})); + ProcessCommand( + PrepareCommand(CmdArgs{"UNSUBSCRIBE", CmdWithArgs::kSubscriberPingChannelName}, ReplyCallback{}) + ); } else if (!strcasecmp(reply_array[0].GetString().c_str(), "UNSUBSCRIBE")) { is_ping_in_flight_ = false; } @@ -804,10 +761,10 @@ void Redis::RedisImpl::FreeCommands() { auto command = commands_.front(); commands_.pop_front(); --commands_size_; - for (const auto& args : command->args.args) { + for (const auto& args : command->args) { InvokeCommandError( command, - args[0], + args.GetCommandName(), ReplyStatus::kEndOfFileError, "Disconnecting, killing commands still waiting in send queue" ); @@ -1091,65 +1048,59 @@ void Redis::RedisImpl::ProcessCommand(const CommandPtr& command) { statistics_.AccountCommandSent(command); bool multi = false; - for (size_t i = 0; i < command->args.args.size(); ++i) { - const auto& args = command->args.args[i]; - const size_t argc = args.size(); - UASSERT(argc >= 1); - if (argc < 1) { - LOG_LIMITED_ERROR() << "Skip empty command to redis"; - continue; - } - - if (IsMultiCommand(args)) multi = true; + for (const auto& args : command->args) { + if (args.IsMultiCommand()) multi = true; if (!context_) { LOG_ERROR() << log_extra_ << "no context"; - InvokeCommandError(command, args[0], ReplyStatus::kOtherError); + InvokeCommandError(command, args.GetCommandName(), ReplyStatus::kOtherError); continue; } - const bool is_special = IsSubscribesCommand(args); + const bool is_special = args.IsSubscribesCommand(); if (is_special) subscriber_ = true; if (subscriber_ && !is_special) { - LOG_ERROR() << log_extra_ << "impossible for subscriber: " << args[0]; - InvokeCommandError(command, args[0], ReplyStatus::kOtherError); + LOG_ERROR() << log_extra_ << "impossible for subscriber: " << args.GetCommandName(); + InvokeCommandError(command, args.GetCommandName(), ReplyStatus::kOtherError); continue; } - if (is_special && (args.size() <= 1 || args[1] != kSubscriberPingChannelName)) { - LOG_INFO() << "Process '" << fmt::to_string(fmt::join(args, " ")) << "' command" << log_extra_; - } - - std::vector argv; - std::vector argv_len; - - argv.reserve(argc); - argv_len.reserve(argc); - - for (const auto& arg : args) { - argv.push_back(arg.data()); - argv_len.push_back(arg.size()); + if (is_special && !args.IsSubscriberPingChannel()) { + LOG_INFO() << "Process '" << fmt::to_string(args.GetJoinedArgs(" ")) << "' command" << log_extra_; } { - if (command->asking && (!multi || IsMultiCommand(args))) { + static constexpr std::size_t kTopArgsCount = 8; + boost::container::small_vector argv; + boost::container::small_vector argv_len; + args.FillPointerSizesStorages(argv, argv_len); + const auto elements_count = argv.size(); + UASSERT(elements_count == argv_len.size()); + UASSERT(elements_count != 0); + + if (command->asking && (!multi || args.IsMultiCommand())) { static const char* asking = "ASKING"; static const size_t asking_len = strlen(asking); redisAsyncCommandArgv(context_, nullptr, nullptr, 1, &asking, &asking_len); } if (redisAsyncCommandArgv( - context_, OnRedisReply, reinterpret_cast(cmd_counter_), argc, argv.data(), argv_len.data() + context_, + OnRedisReply, + reinterpret_cast(cmd_counter_), + elements_count, + argv.data(), + argv_len.data() ) != REDIS_OK) { - LOG_ERROR() << log_extra_ << "redisAsyncCommandArgv() failed on command " << args[0]; - InvokeCommandError(command, args[0], ReplyStatus::kOtherError); + LOG_ERROR() << log_extra_ << "redisAsyncCommandArgv() failed on command " << args.GetCommandName(); + InvokeCommandError(command, args.GetCommandName(), ReplyStatus::kOtherError); continue; } } - if (IsExecCommand(args)) multi = false; + if (args.IsExecCommand()) multi = false; - if (!IsUnsubscribeCommand(args)) { + if (!args.IsUnsubscribeCommand()) { auto entry = std::make_unique(); - entry->cmd = args[0]; + entry->cmd = args.GetCommandName(); entry->meta = command; entry->timer.data = this; entry->redis_impl = shared_from_this(); diff --git a/redis/src/storages/redis/impl/redis_stats.cpp b/redis/src/storages/redis/impl/redis_stats.cpp index 09a35091f6d3..939a95e0102c 100644 --- a/redis/src/storages/redis/impl/redis_stats.cpp +++ b/redis/src/storages/redis/impl/redis_stats.cpp @@ -153,9 +153,8 @@ void Statistics::AccountStateChanged(RedisState new_state) { } void Statistics::AccountCommandSent(const CommandPtr& cmd) { - for (const auto& args : cmd->args.args) { - size_t size = 0; - for (const auto& arg : args) size += arg.size(); + for (const auto& args : cmd->args) { + std::size_t size = args.GetCommandBytesLength(); request_size_percentile.GetCurrentCounter().Account(size); } } diff --git a/redis/src/storages/redis/impl/request.cpp b/redis/src/storages/redis/impl/request.cpp index 308f0be12ae6..ff9735e4f647 100644 --- a/redis/src/storages/redis/impl/request.cpp +++ b/redis/src/storages/redis/impl/request.cpp @@ -40,15 +40,15 @@ class ReplyState { }; std::string MakeSpanName(const CmdArgs& cmd_args) { - if (cmd_args.args.empty() || cmd_args.args.front().empty()) { + if (cmd_args.GetCommandCount() == 0) { return "redis_unknown"; } - if (cmd_args.args.size() > 1) { + if (cmd_args.GetCommandCount() > 1) { return "redis_multi"; } - return "redis_" + cmd_args.args.front().front(); + return "redis_" + cmd_args.GetCommandName(0); } } // namespace diff --git a/redis/src/storages/redis/impl/sentinel_impl.cpp b/redis/src/storages/redis/impl/sentinel_impl.cpp index 8c385bdd1857..3cda5f158768 100644 --- a/redis/src/storages/redis/impl/sentinel_impl.cpp +++ b/redis/src/storages/redis/impl/sentinel_impl.cpp @@ -41,11 +41,11 @@ struct CommandSpecialPrinter { logging::LogHelper& operator<<(logging::LogHelper& os, CommandSpecialPrinter v) { const auto& command = v.command; - if (command->args.args.size() == 1 || command->invoke_counter + 1 >= command->args.args.size()) { + if (command->args.GetCommandCount() == 1 || command->invoke_counter + 1 >= command->args.GetCommandCount()) { os << command->args; - } else if (command->invoke_counter < command->args.args.size() && !command->args.args[command->invoke_counter].empty()) { + } else if (command->invoke_counter < command->args.GetCommandCount()) { os << fmt::format( - "subrequest idx={}, cmd={}", command->invoke_counter, command->args.args[command->invoke_counter].front() + "subrequest idx={}, cmd={}", command->invoke_counter, command->args.GetCommandName(command->invoke_counter) ); } @@ -499,10 +499,10 @@ void SentinelImpl::Stop() { std::lock_guard lock(command_mutex_); while (!commands_.empty()) { auto command = commands_.back().command; - for (const auto& args : command->args.args) { - LOG_ERROR() << fmt::format("Killing request: {}", fmt::join(args, ", ")); + for (const auto& args : command->args) { + LOG_ERROR() << fmt::format("Killing request: {}", args.GetJoinedArgs(", ")); auto reply = std::make_shared( - args[0], + args.GetCommandName(), nullptr, ReplyStatus::kEndOfFileError, "Stopping, killing commands remaining in send queue" @@ -843,9 +843,9 @@ void SentinelImpl::ProcessWaitingCommands() { const auto& command = scommand.command; const CommandControlImpl cc{command->control}; if (scommand.start + cc.timeout_all < now) { - for (const auto& args : command->args.args) { + for (const auto& args : command->args) { auto reply = std::make_shared( - args[0], nullptr, ReplyStatus::kTimeoutError, "Command in the send queue timed out" + args.GetCommandName(), nullptr, ReplyStatus::kTimeoutError, "Command in the send queue timed out" ); statistics_internal_.redis_not_ready++; InvokeCommand(command, std::move(reply)); diff --git a/redis/src/storages/redis/impl/sentinel_query.hpp b/redis/src/storages/redis/impl/sentinel_query.hpp index 84219ba05ca5..1a610ea16a6a 100644 --- a/redis/src/storages/redis/impl/sentinel_query.hpp +++ b/redis/src/storages/redis/impl/sentinel_query.hpp @@ -3,6 +3,7 @@ #include #include +#include #include "shard.hpp" USERVER_NAMESPACE_BEGIN @@ -15,14 +16,22 @@ struct GetHostsRequest { : sentinel_shard(sentinel_shard), command({"SENTINEL", "MASTERS"}), master(true), - password(std::move(password)) {} + password(std::move(password)) { + UASSERT(command.GetCommandCount() == 1); + UASSERT_MSG( + fmt::to_string(command.begin()->GetJoinedArgs(";")) == "SENTINEL;MASTERS", + fmt::to_string(command.begin()->GetJoinedArgs(";")) + ); + } // For SLAVES GetHostsRequest(Shard& sentinel_shard, std::string shard_name, Password password) : sentinel_shard(sentinel_shard), command({"SENTINEL", "SLAVES", std::move(shard_name)}), master(false), - password(std::move(password)) {} + password(std::move(password)) { + UASSERT(command.GetCommandCount() == 1); + } Shard& sentinel_shard; CmdArgs command; diff --git a/scripts/docs/en/userver/build/options.md b/scripts/docs/en/userver/build/options.md index c4f2401c39bc..2fca5de819e3 100644 --- a/scripts/docs/en/userver/build/options.md +++ b/scripts/docs/en/userver/build/options.md @@ -181,7 +181,7 @@ The exact format of setting cmake options varies depending on the method of buil It is possible to build userver based services with libraries statically linked in. -@warning The support is platform dependant, as a result some libraies on some platforms may linked dynamically. Feel free to provide a PR to support your favourite platform. +@warning The support is platform dependant, as a result some libraries on some platforms may linked dynamically. Feel free to provide a PR to support your favourite platform. Userver does not build dynamic libraries itself, but most of its dependencies do. CMake (by default) prefers dynamic libraries on Unix-like operating systems. @@ -190,7 +190,7 @@ With the option, CMake tries to find all dependencies as static libraries (and d - To link statically with `libstdc++` or `libc++`, use `CMAKE_EXE_LINKER_FLAGS="-static-libstdc++ -static-libgcc"`. - To force fully static binary (with statically linked `libc`), use `CMAKE_EXE_LINKER_FLAGS="-static"`. In such case, all dependencies must be provided as static libraries. Also `userver` must be build with `USERVER_DISABLE_PHDR_CACHE=ON` (without this flag, it can lead to endless memory allocation). -Some dependecies usually should be build from source for statically linked executable: +Some dependencies usually should be build from source for statically linked executable: 1. `Curl`. Use `USERVER_FORCE_DOWNLOAD_CURL=ON` to download and build Curl from source. 2. `cctz`, `yaml-cpp`, `fmt` often have no static libraries in their packages, so they should be build from source and installed in your host system (for instance, in `/usr/local`). diff --git a/scripts/docs/en/userver/roadmap_and_changelog.md b/scripts/docs/en/userver/roadmap_and_changelog.md index 82295f73ef12..e451c8ee3991 100644 --- a/scripts/docs/en/userver/roadmap_and_changelog.md +++ b/scripts/docs/en/userver/roadmap_and_changelog.md @@ -20,19 +20,71 @@ Changelog news also go to the * ✔️ HTTP 2.0 server support * ✔️ Improve OpenTelemetry Protocol (OTLP) support. * ✔️ Improve Kafka driver. +* ✔️ Logging format customization, including JSON logging. +* ✔️ Secdist simplification and functionality improvement. +* ✔️ Improved MacOS build support. +* ✔️ Improved Conan support. * 👨‍💻 gRPC simplification and functionality improvement. -* 👨‍💻 Logging format customization, including JSON logging. -* 👨‍💻 Secdist simplification and functionality improvement. * 👨‍💻 Add retry budget or retry circuit breaker for clients. -* Improved MacOS build support. -* Add web interface to the [uservice-dynconf](https://github.com/userver-framework/uservice-dynconf) -* Generate full-blown accessories for OpenAPI: +* 👨‍💻 Generate full-blown accessories for OpenAPI: * clients * handlers +* Add web interface to the [uservice-dynconf](https://github.com/userver-framework/uservice-dynconf) ## Changelog +### Release v2.7 + +* Logging in JSON format was implemented. See static option `format` at components::Logging. +* utils::regex now uses `Re2` under the hood, leading to at least x2 faster regular expression matching and guaranteed + absence of backtracking. Updating is highly recommended. +* Mongo connection state checking algorithms was adjusted to work well on small RPS. +* Conan packages now support all the userver features. Conan package build now reuses the CMake install targets and + CMake config files. +* Full feature support for MacOS, including testing and Conan package build and usage on that platform. +* Added support for TLS certificate chains. See `tls.cert` static option at components::Server. Many thanks to + [aklyuchev](https://github.com/aklyuchev) for the PR! +* Chaotic exceptions now do not depend on JSON. Thanks to [Artyom](https://github.com/Lookingforcommit) for the PR! + +* gRPC + * Retries are now supported and controlled via dynamic config. See ugrpc::client::Qos for more info. + * Out-the-box cache dump support for Protobuf messages. See + @ref dump_serialization_guide "Implementing serialization (Write / Read)" for more info. + * Removed deprecated `*Sync` methods. + +* Optimizations + * Speed up configuration reads on creating new PostgreSQL connections. + * utils::PeriodicTask now calls RCU Read two times less on each iteration. + * Reduced memory allocations count on each Redis request. + +* Build + * Fixed build with `USERVER_FEATURE_JEMALLOC=ON`. Many thanks to [Aleksey Ignatiev](https://github.com/ae-ignatiev) + for the PR! + * Service templates [service_template](https://github.com/userver-framework/service_template), + [pg_service_template](https://github.com/userver-framework/pg_service_template), + [pg_grpc_service_template](https://github.com/userver-framework/pg_grpc_service_template), + [mongo_grpc_service_template](https://github.com/userver-framework/mongo_grpc_service_template) now use + @ref service_templates_presets "cmake presets" and @ref devcontainers "devcontainers" for out-of-the-box support + of VSCode and Clion IDEs. + * Started the work on Ubuntu 24.04 images. + * Added `ubuntu-22.04-userver-pg-dev` image with all the tools for development. Planning to switch to Ubuntu-24.04 and + leave only 2 containers: with build dependencies to build userver, and with prebuild userver. + * Added missing fmt11 headers. Thanks to [Pavel Sidorovich](https://github.com/RayeS2070) for the PR! + * Added `USERVER_USE_STATIC_LIBS` to link third-party libraries statically. + * Support `pacman` epoch in CMake version detection. Many thanks to [Konstantin Goncharik](https://github.com/botanegg) + for the PR. + +* Documentation + * Significant update of the @ref scripts/docs/en/userver/build/build.md + * More docs for tracing::Span::SetLogLevel() and tracing::Span::SetLocalLogLevel() + * Fixed secdist example at components::Mongo. Thank to [Nikita Puteev](https://github.com/Malfak) for the PR! + * Highlight the functionality of formats::common::Item in each supported format. + * Add info about full static linkage. Thanks to [Nikita](https://github.com/root-kidik) for the PR! + * Better `runtests` documentation at @ref scripts/docs/en/userver/functional_testing.md + * Documentation and samples for storages::postgres::io::Codegen{}. + + ### Release v2.6 * storages::secdist::Secdist is now automatically reloaded for Mongo, Redis and PostgreSQL databases if the secdist file diff --git a/universal/include/userver/crypto/certificate.hpp b/universal/include/userver/crypto/certificate.hpp index c3e75926dc69..9e15bf885d3f 100644 --- a/universal/include/userver/crypto/certificate.hpp +++ b/universal/include/userver/crypto/certificate.hpp @@ -52,7 +52,7 @@ using CertificatesChain = std::list; /// list of 'Certificate's. /// /// @throw crypto::KeyParseError if failed to load the certificate. -CertificatesChain LoadCertficatesChainFromString(std::string_view chain); +CertificatesChain LoadCertificatesChainFromString(std::string_view chain); } // namespace crypto diff --git a/universal/include/userver/formats/serialize/write_to_stream.hpp b/universal/include/userver/formats/serialize/write_to_stream.hpp index 0a889fa262c4..c424fcc4249a 100644 --- a/universal/include/userver/formats/serialize/write_to_stream.hpp +++ b/universal/include/userver/formats/serialize/write_to_stream.hpp @@ -14,6 +14,10 @@ #include +namespace boost::uuids { +struct uuid; +} + USERVER_NAMESPACE_BEGIN namespace utils::impl::strong_typedef { @@ -85,6 +89,10 @@ WriteToStream(const T& value, StringBuilder& sw) { if constexpr (meta::kIsMap) { impl::WriteToStreamDict(value, sw); } else if constexpr (meta::kIsRange) { + static_assert( + !std::is_same_v, + "Include to serialize 'boost::uuids::uuid" + ); static_assert( !meta::kIsRecursiveRange, "Trying to log a recursive range, which can be dangerous. " diff --git a/universal/include/userver/utils/small_string.hpp b/universal/include/userver/utils/small_string.hpp index 2db931836416..7df512acd827 100644 --- a/universal/include/userver/utils/small_string.hpp +++ b/universal/include/userver/utils/small_string.hpp @@ -142,6 +142,10 @@ class SmallString final { /// @brief Append contents of a string_view to the string. void append(std::string_view str); + /// @brief Inserts elements from range [begin, end) before pos. + template + void insert(const_iterator pos, InputIt begin, InputIt end); + /// @brief Remove the last character from the string. void pop_back(); @@ -276,6 +280,12 @@ void SmallString::append(std::string_view str) { data_.insert(data_.begin() + old_size, str.begin(), str.end()); } +template +template +void SmallString::insert(SmallString::const_iterator pos, InputIt begin, InputIt end) { + data_.insert(pos, begin, end); +} + template void SmallString::pop_back() { data_.pop_back(); diff --git a/universal/src/crypto/certificate.cpp b/universal/src/crypto/certificate.cpp index ef79d807ebf5..dd100c054650 100644 --- a/universal/src/crypto/certificate.cpp +++ b/universal/src/crypto/certificate.cpp @@ -56,7 +56,7 @@ Certificate Certificate::LoadFromString(std::string_view certificate) { return Certificate{std::move(cert)}; } -CertificatesChain LoadCertficatesChainFromString(std::string_view chain) { +CertificatesChain LoadCertificatesChainFromString(std::string_view chain) { CertificatesChain certificates; constexpr std::string_view kBeginMarker = "-----BEGIN CERTIFICATE-----"; constexpr std::string_view kEndMarker = "-----END CERTIFICATE-----"; diff --git a/universal/src/utils/small_string_test.cpp b/universal/src/utils/small_string_test.cpp index aa512fed5439..2ee17c94c902 100644 --- a/universal/src/utils/small_string_test.cpp +++ b/universal/src/utils/small_string_test.cpp @@ -69,6 +69,14 @@ TEST(SmallString, Append) { EXPECT_EQ(str, "abcdabcd"); } +TEST(SmallString, Insert) { + constexpr std::string_view data{"ab"}; + + utils::SmallString<3> str("c"); + str.insert(str.begin(), data.begin(), data.end()); + EXPECT_EQ(str, "abc"); +} + TEST(SmallString, SizeCapacity) { utils::SmallString<10> str("abcd"); str.resize(3, '1'); @@ -153,6 +161,12 @@ TEST(SmallString, Format) { EXPECT_EQ("abcd1234", fmt::format("{}{}", str1, str2)); } +TEST(SmallString, FormatTo) { + utils::SmallString<6> str("abc"); + fmt::format_to(std::back_inserter(str), "d={}", 1); + EXPECT_EQ("abcd=1", str); +} + TEST(SmallString, Iterator) { utils::SmallString<3> str("12345"); for (auto& c : str) c++;