feat logging: improve LogCaptureFixture

* Parse text and tags from logs, accounting for escaping perfectly
* Allow handy searching for logs
* Allow mocking non-default loggers
* Add `utils::encoding::TskvParser`
32120b023c37fa96399f5197931c011202112705

.clang-tidy

@@ -97,6 +97,8 @@ Checks:
   ,-readability-redundant-declaration\
   ,-readability-uppercase-literal-suffix\
   ,-readability-use-anyofallof\
+  ,-hicpp-use-emplace\
+  ,-modernize-use-emplace\
   "
 
 WarningsAsErrors: '*'

.mapping.json

@@ -750,7 +750,6 @@
   "core/include/userver/utils/impl/wait_token_storage.hpp":"taxi/uservices/userver/core/include/userver/utils/impl/wait_token_storage.hpp",
   "core/include/userver/utils/impl/wrapped_call.hpp":"taxi/uservices/userver/core/include/userver/utils/impl/wrapped_call.hpp",
   "core/include/userver/utils/impl/wrapped_call_base.hpp":"taxi/uservices/userver/core/include/userver/utils/impl/wrapped_call_base.hpp",
-  "core/include/userver/utils/internal_tag_fwd.hpp":"taxi/uservices/userver/core/include/userver/utils/internal_tag_fwd.hpp",
   "core/include/userver/utils/lazy_shared_ptr.hpp":"taxi/uservices/userver/core/include/userver/utils/lazy_shared_ptr.hpp",
   "core/include/userver/utils/log.hpp":"taxi/uservices/userver/core/include/userver/utils/log.hpp",
   "core/include/userver/utils/periodic_task.hpp":"taxi/uservices/userver/core/include/userver/utils/periodic_task.hpp",
@@ -1531,7 +1530,6 @@
   "core/src/utils/impl/wait_token_storage.cpp":"taxi/uservices/userver/core/src/utils/impl/wait_token_storage.cpp",
   "core/src/utils/impl/wait_token_storage_test.cpp":"taxi/uservices/userver/core/src/utils/impl/wait_token_storage_test.cpp",
   "core/src/utils/impl/wrapped_call_base.cpp":"taxi/uservices/userver/core/src/utils/impl/wrapped_call_base.cpp",
-  "core/src/utils/internal_tag.hpp":"taxi/uservices/userver/core/src/utils/internal_tag.hpp",
   "core/src/utils/jemalloc.cpp":"taxi/uservices/userver/core/src/utils/jemalloc.cpp",
   "core/src/utils/jemalloc.hpp":"taxi/uservices/userver/core/src/utils/jemalloc.hpp",
   "core/src/utils/lazy_shared_ptr_test.cpp":"taxi/uservices/userver/core/src/utils/lazy_shared_ptr_test.cpp",
@@ -3670,6 +3668,8 @@
   "universal/include/userver/utils/distances.hpp":"taxi/uservices/userver/universal/include/userver/utils/distances.hpp",
   "universal/include/userver/utils/encoding/hex.hpp":"taxi/uservices/userver/universal/include/userver/utils/encoding/hex.hpp",
   "universal/include/userver/utils/encoding/tskv.hpp":"taxi/uservices/userver/universal/include/userver/utils/encoding/tskv.hpp",
+  "universal/include/userver/utils/encoding/tskv_parser.hpp":"taxi/uservices/userver/universal/include/userver/utils/encoding/tskv_parser.hpp",
+  "universal/include/userver/utils/encoding/tskv_parser_read.hpp":"taxi/uservices/userver/universal/include/userver/utils/encoding/tskv_parser_read.hpp",
   "universal/include/userver/utils/enumerate.hpp":"taxi/uservices/userver/universal/include/userver/utils/enumerate.hpp",
   "universal/include/userver/utils/exception.hpp":"taxi/uservices/userver/universal/include/userver/utils/exception.hpp",
   "universal/include/userver/utils/expected.hpp":"taxi/uservices/userver/universal/include/userver/utils/expected.hpp",
@@ -3684,13 +3684,13 @@
   "universal/include/userver/utils/get_if.hpp":"taxi/uservices/userver/universal/include/userver/utils/get_if.hpp",
   "universal/include/userver/utils/impl/boost_variadic_to_seq.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/boost_variadic_to_seq.hpp",
   "universal/include/userver/utils/impl/disable_core_dumps.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/disable_core_dumps.hpp",
+  "universal/include/userver/utils/impl/internal_tag.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/internal_tag.hpp",
   "universal/include/userver/utils/impl/internal_tag_fwd.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/internal_tag_fwd.hpp",
   "universal/include/userver/utils/impl/intrusive_link_mode.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/intrusive_link_mode.hpp",
   "universal/include/userver/utils/impl/projecting_view.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/projecting_view.hpp",
   "universal/include/userver/utils/impl/source_location.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/source_location.hpp",
   "universal/include/userver/utils/impl/static_registration.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/static_registration.hpp",
   "universal/include/userver/utils/impl/transparent_hash.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/transparent_hash.hpp",
-  "universal/include/userver/utils/impl/weak_internal_tag.hpp":"taxi/uservices/userver/universal/include/userver/utils/impl/weak_internal_tag.hpp",
   "universal/include/userver/utils/invariant_error.hpp":"taxi/uservices/userver/universal/include/userver/utils/invariant_error.hpp",
   "universal/include/userver/utils/ip.hpp":"taxi/uservices/userver/universal/include/userver/utils/ip.hpp",
   "universal/include/userver/utils/lazy_prvalue.hpp":"taxi/uservices/userver/universal/include/userver/utils/lazy_prvalue.hpp",
@@ -3952,6 +3952,8 @@
   "universal/src/utils/encoding/hex.cpp":"taxi/uservices/userver/universal/src/utils/encoding/hex.cpp",
   "universal/src/utils/encoding/hex_benchmark.cpp":"taxi/uservices/userver/universal/src/utils/encoding/hex_benchmark.cpp",
   "universal/src/utils/encoding/hex_test.cpp":"taxi/uservices/userver/universal/src/utils/encoding/hex_test.cpp",
+  "universal/src/utils/encoding/tskv_parser.cpp":"taxi/uservices/userver/universal/src/utils/encoding/tskv_parser.cpp",
+  "universal/src/utils/encoding/tskv_parser_test.cpp":"taxi/uservices/userver/universal/src/utils/encoding/tskv_parser_test.cpp",
   "universal/src/utils/encoding/tskv_test.cpp":"taxi/uservices/userver/universal/src/utils/encoding/tskv_test.cpp",
   "universal/src/utils/encoding/tskv_testdata_bin.hpp":"taxi/uservices/userver/universal/src/utils/encoding/tskv_testdata_bin.hpp",
   "universal/src/utils/enumerate_test.cpp":"taxi/uservices/userver/universal/src/utils/enumerate_test.cpp",
@@ -4047,6 +4049,7 @@
   "universal/utest/src/utest/assert_macros_test.cpp":"taxi/uservices/userver/universal/utest/src/utest/assert_macros_test.cpp",
   "universal/utest/src/utest/current_process_open_files.cpp":"taxi/uservices/userver/universal/utest/src/utest/current_process_open_files.cpp",
   "universal/utest/src/utest/current_process_open_files_test.cpp":"taxi/uservices/userver/universal/utest/src/utest/current_process_open_files_test.cpp",
+  "universal/utest/src/utest/log_capture_fixture.cpp":"taxi/uservices/userver/universal/utest/src/utest/log_capture_fixture.cpp",
   "universal/utest/src/utest/parameter_names_test.cpp":"taxi/uservices/userver/universal/utest/src/utest/parameter_names_test.cpp",
   "ydb/CMakeLists.txt":"taxi/uservices/userver/ydb/CMakeLists.txt",
   "ydb/functional_tests/CMakeLists.txt":"taxi/uservices/userver/ydb/functional_tests/CMakeLists.txt",

chaotic/integration_tests/tests/render/logging.cpp

@@ -13,20 +13,18 @@ using Logging = utest::LogCaptureFixture<>;
 
 TEST_F(Logging, Object) {
   auto json = formats::json::MakeObject(
-      "integer", 1, "boolean", true, "number", 1.1, "string", "foo",
+      "integer", 1, "boolean", true, "number", 1.5, "string", "foo",
       "string-enum", "1", "object", formats::json::MakeObject(), "array",
       formats::json::MakeArray(1));
   auto obj = json.As<ns::ObjectTypes>();
 
-  LOG_INFO() << obj;
-  ASSERT_THAT(
-      ExtractRawLog(),
-      testing::HasSubstr(
-          "text={\"boolean\":true,\"integer\":1,\"number\":1.1,\"string\":"
-          "\"foo\",\"object\":{},\"array\":[1],\"string-enum\":\"1\"}\n"));
+  const auto obj_string = GetLogCapture().ToStringViaLogging(obj);
+  EXPECT_EQ(obj_string,
+            R"({"boolean":true,"integer":1,"number":1.5,"string":"foo",)"
+            R"("object":{},"array":[1],"string-enum":"1"})");
 
-  LOG_INFO() << obj.string_enum;
-  ASSERT_THAT(ExtractRawLog(), testing::HasSubstr("\ttext=1\n"));
+  const auto enum_string = GetLogCapture().ToStringViaLogging(obj.string_enum);
+  ASSERT_EQ(enum_string, "1");
 }
 
 USERVER_NAMESPACE_END

core/include/userver/cache/cache_statistics.hpp

@@ -8,7 +8,7 @@
 #include <cstddef>
 
 #include <userver/cache/update_type.hpp>
-#include <userver/utils/internal_tag_fwd.hpp>
+#include <userver/utils/impl/internal_tag.hpp>
 #include <userver/utils/statistics/fwd.hpp>
 #include <userver/utils/statistics/rate_counter.hpp>
 
@@ -62,7 +62,8 @@ class UpdateStatisticsScope final {
 
   ~UpdateStatisticsScope();
 
-  impl::UpdateState GetState(utils::InternalTag) const;
+  // For internal use only
+  impl::UpdateState GetState(utils::impl::InternalTag) const;
   /// @endcond
 
   /// @brief Mark that the `Update` has finished with changes

core/include/userver/dynamic_config/updater/component.hpp

@@ -21,7 +21,6 @@
 #include <userver/dynamic_config/updates_sink/component.hpp>
 #include <userver/engine/mutex.hpp>
 #include <userver/utils/impl/transparent_hash.hpp>
-#include <userver/utils/internal_tag_fwd.hpp>
 
 USERVER_NAMESPACE_BEGIN
 

core/include/userver/dynamic_config/value.hpp

@@ -8,8 +8,8 @@
 #include <userver/formats/parse/common_containers.hpp>
 #include <userver/formats/serialize/common_containers.hpp>
 #include <userver/utils/default_dict.hpp>
+#include <userver/utils/impl/internal_tag.hpp>
 #include <userver/utils/impl/transparent_hash.hpp>
-#include <userver/utils/internal_tag_fwd.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -35,14 +35,16 @@ class DocsMap final {
   bool AreContentsEqual(const DocsMap& other) const;
 
   /// @cond
-  // For internal use only
+  // For internal use only.
   // Set of configs expected to be used is automatically updated when
   // configs are retrieved with 'Get' method.
   void SetConfigsExpectedToBeUsed(
-      utils::impl::TransparentSet<std::string> configs, utils::InternalTag);
+      utils::impl::TransparentSet<std::string> configs,
+      utils::impl::InternalTag);
 
+  // For internal use only.
   const utils::impl::TransparentSet<std::string>& GetConfigsExpectedToBeUsed(
-      utils::InternalTag) const;
+      utils::impl::InternalTag) const;
   /// @endcond
 
  private:

core/include/userver/os_signals/processor.hpp

@@ -6,7 +6,7 @@
 #include <userver/concurrent/async_event_channel.hpp>
 #include <userver/engine/task/task_processor_fwd.hpp>
 #include <userver/os_signals/subscriber.hpp>
-#include <userver/utils/internal_tag_fwd.hpp>
+#include <userver/utils/impl/internal_tag.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -64,7 +64,7 @@ class Processor final {
 
   /// @cond
   // For internal use
-  void Notify(int signum, utils::InternalTag);
+  void Notify(int signum, utils::impl::InternalTag);
   /// @endcond
  private:
   concurrent::AsyncEventChannel<int> channel_;

core/include/userver/tracing/span.hpp

@@ -10,8 +10,8 @@
 #include <userver/logging/log_extra.hpp>
 #include <userver/tracing/scope_time.hpp>
 #include <userver/tracing/tracer_fwd.hpp>
+#include <userver/utils/impl/internal_tag.hpp>
 #include <userver/utils/impl/source_location.hpp>
-#include <userver/utils/internal_tag_fwd.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -206,9 +206,11 @@ class Span final {
   std::chrono::system_clock::time_point GetStartSystemTime() const;
 
   /// @cond
-  void AddTags(const logging::LogExtra&, utils::InternalTag);
+  // For internal use only.
+  void AddTags(const logging::LogExtra&, utils::impl::InternalTag);
 
-  impl::TimeStorage& GetTimeStorage();
+  // For internal use only.
+  impl::TimeStorage& GetTimeStorage(utils::impl::InternalTag);
 
   // For internal use only.
   void LogTo(logging::impl::TagWriter writer) const&;

core/src/cache/cache_statistics.cpp

@@ -3,7 +3,6 @@
 #include <userver/utils/assert.hpp>
 #include <userver/utils/datetime.hpp>
 #include <userver/utils/statistics/writer.hpp>
-#include <utils/internal_tag.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -123,7 +122,8 @@ UpdateStatisticsScope::~UpdateStatisticsScope() {
   }
 }
 
-impl::UpdateState UpdateStatisticsScope::GetState(utils::InternalTag) const {
+impl::UpdateState UpdateStatisticsScope::GetState(
+    utils::impl::InternalTag) const {
   return state_;
 }
 

core/src/cache/cache_update_trait_impl.cpp

@@ -20,7 +20,6 @@
 #include <dump/dump_locator.hpp>
 #include <userver/dump/factory.hpp>
 #include <userver/testsuite/testsuite_support.hpp>
-#include <utils/internal_tag.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -403,7 +402,8 @@ void CacheUpdateTrait::Impl::DoUpdate(UpdateType update_type,
 
   try {
     customized_trait_.Update(update_type, last_update_, now, stats);
-    CheckUpdateState(stats.GetState(utils::InternalTag{}), update_type_str);
+    CheckUpdateState(stats.GetState(utils::impl::InternalTag{}),
+                     update_type_str);
   } catch (const std::exception& e) {
     OnUpdateFailure(config);
     throw;

core/src/components/manager.cpp

@@ -23,7 +23,6 @@
 #include <userver/os_signals/component.hpp>
 #include <userver/utils/async.hpp>
 #include <userver/utils/distances.hpp>
-#include <utils/internal_tag.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -233,7 +232,7 @@ void Manager::OnSignal(int signum) {
   std::shared_lock<std::shared_timed_mutex> lock(context_mutex_);
   if (components_cleared_) return;
   if (signal_processor_) {
-    signal_processor_->Get().Notify(signum, utils::InternalTag{});
+    signal_processor_->Get().Notify(signum, utils::impl::InternalTag{});
   }
 }
 

core/src/dynamic_config/storage/component.cpp

@@ -26,7 +26,6 @@
 #include <userver/yaml_config/merge_schemas.hpp>
 
 #include <dynamic_config/storage_data.hpp>
-#include <utils/internal_tag.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -199,9 +198,9 @@ dynamic_config::impl::SnapshotData DynamicConfig::Impl::ParseConfig(
 void DynamicConfig::Impl::DoSetConfig(const dynamic_config::DocsMap& value) {
   auto config = ParseConfig(value);
 
-  if (!value.GetConfigsExpectedToBeUsed(utils::InternalTag{}).empty()) {
+  if (!value.GetConfigsExpectedToBeUsed(utils::impl::InternalTag{}).empty()) {
     LOG_INFO() << "Some configs expected to be used are actually not needed: "
-               << value.GetConfigsExpectedToBeUsed(utils::InternalTag{});
+               << value.GetConfigsExpectedToBeUsed(utils::impl::InternalTag{});
   }
 
   auto after_assign_hook = [&] {

core/src/dynamic_config/updater/component.cpp

@@ -12,7 +12,6 @@
 #include <userver/utils/fast_scope_guard.hpp>
 #include <userver/utils/string_to_duration.hpp>
 #include <userver/yaml_config/merge_schemas.hpp>
-#include <utils/internal_tag.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -94,7 +93,8 @@ dynamic_config::DocsMap DynamicConfigClientUpdater::MergeDocsMap(
     const std::vector<std::string>& removed) {
   dynamic_config::DocsMap combined(std::move(update));
   combined.MergeMissing(current);
-  combined.SetConfigsExpectedToBeUsed(docs_map_keys_, utils::InternalTag{});
+  combined.SetConfigsExpectedToBeUsed(docs_map_keys_,
+                                      utils::impl::InternalTag{});
   for (const auto& key : removed) {
     combined.Remove(key);
   }

core/src/dynamic_config/value.cpp

@@ -6,7 +6,6 @@
 
 #include <userver/formats/json/serialize.hpp>
 #include <userver/formats/json/value_builder.hpp>
-#include <utils/internal_tag.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -83,12 +82,13 @@ bool DocsMap::AreContentsEqual(const DocsMap& other) const {
 }
 
 void DocsMap::SetConfigsExpectedToBeUsed(
-    utils::impl::TransparentSet<std::string> configs, utils::InternalTag) {
+    utils::impl::TransparentSet<std::string> configs,
+    utils::impl::InternalTag) {
   configs_to_be_used_ = std::move(configs);
 }
 
 const utils::impl::TransparentSet<std::string>&
-DocsMap::GetConfigsExpectedToBeUsed(utils::InternalTag) const {
+DocsMap::GetConfigsExpectedToBeUsed(utils::impl::InternalTag) const {
   return configs_to_be_used_;
 }
 

core/src/dynamic_config/value_test.cpp

@@ -6,7 +6,6 @@
 #include <userver/formats/json/value.hpp>
 #include <userver/formats/json/value_builder.hpp>
 #include <userver/formats/yaml/value.hpp>
-#include <utils/internal_tag.hpp>
 
 USERVER_NAMESPACE_BEGIN
 
@@ -21,15 +20,15 @@ TEST(DocsMap, HasConfig) {
 
 TEST(DocsMap, AreContentsEqualTrue) {
   dynamic_config::DocsMap docs_map1;
-  docs_map1.SetConfigsExpectedToBeUsed({"a"}, utils::InternalTag{});
+  docs_map1.SetConfigsExpectedToBeUsed({"a"}, utils::impl::InternalTag{});
   docs_map1.Parse(R"({"a": "a", "b": "b"})", false);
 
   dynamic_config::DocsMap docs_map2;
-  docs_map2.SetConfigsExpectedToBeUsed({"b"}, utils::InternalTag{});
+  docs_map2.SetConfigsExpectedToBeUsed({"b"}, utils::impl::InternalTag{});
   docs_map2.Parse(R"({"b": "b", "a": "a"})", false);
 
-  EXPECT_NE(docs_map1.GetConfigsExpectedToBeUsed(utils::InternalTag{}),
-            docs_map2.GetConfigsExpectedToBeUsed(utils::InternalTag{}));
+  EXPECT_NE(docs_map1.GetConfigsExpectedToBeUsed(utils::impl::InternalTag{}),
+            docs_map2.GetConfigsExpectedToBeUsed(utils::impl::InternalTag{}));
   EXPECT_TRUE(docs_map1.AreContentsEqual(docs_map2));
 }
 
@@ -40,8 +39,8 @@ TEST(DocsMap, AreContentsEqualFalse) {
   dynamic_config::DocsMap docs_map2;
   docs_map2.Parse(R"({"a": "b", "b": "a"})", false);
 
-  EXPECT_EQ(docs_map1.GetConfigsExpectedToBeUsed(utils::InternalTag{}),
-            docs_map2.GetConfigsExpectedToBeUsed(utils::InternalTag{}));
+  EXPECT_EQ(docs_map1.GetConfigsExpectedToBeUsed(utils::impl::InternalTag{}),
+            docs_map2.GetConfigsExpectedToBeUsed(utils::impl::InternalTag{}));
   EXPECT_FALSE(docs_map1.AreContentsEqual(docs_map2));
 }
 
@@ -50,22 +49,23 @@ TEST(DocsMap, ConfigExpectedToBeUsedRemovedAfterGet) {
   utils::impl::TransparentSet<std::string> to_be_used = {"a", "b"};
   docs_map.SetConfigsExpectedToBeUsed(
       utils::impl::TransparentSet<std::string>(to_be_used),
-      utils::InternalTag{});
+      utils::impl::InternalTag{});
   docs_map.Parse(R"({"a": "a", "b": "b"})", false);
 
-  EXPECT_EQ(docs_map.GetConfigsExpectedToBeUsed(utils::InternalTag{}),
+  EXPECT_EQ(docs_map.GetConfigsExpectedToBeUsed(utils::impl::InternalTag{}),
             to_be_used);
 
   (void)docs_map.Get("a");
 
-  EXPECT_EQ(docs_map.GetConfigsExpectedToBeUsed(utils::InternalTag{}),
+  EXPECT_EQ(docs_map.GetConfigsExpectedToBeUsed(utils::impl::InternalTag{}),
             utils::impl::TransparentSet<std::string>({"b"}));
 
   dynamic_config::DocsMap docs_map_copy(docs_map);
   (void)docs_map_copy.Get("b");
 
   EXPECT_TRUE(
-      docs_map_copy.GetConfigsExpectedToBeUsed(utils::InternalTag{}).empty());
+      docs_map_copy.GetConfigsExpectedToBeUsed(utils::impl::InternalTag{})
+          .empty());
 }
 
 TEST(DocsMap, Merge) {

core/src/os_signals/component.cpp

@@ -2,8 +2,6 @@
 
 #include <userver/engine/task/task_base.hpp>
 
-#include <utils/internal_tag.hpp>
-
 USERVER_NAMESPACE_BEGIN
 
 namespace os_signals {