Merge branch 'master' of github.com:realm/realm-core into ct/capi_ret…

…rieve_persisted_schema_version

CHANGELOG.md

@@ -1,9 +1,31 @@
 # NEXT RELEASE
 
 ### Enhancements
-* <New feature description> (PR [#????](https://github.com/realm/realm-core/pull/????))
+* On Windows devices Device Sync will additionally look up SSL certificates in the Windows Trusted Root Certification Authorities certificate store when establishing a connection. (PR [#7882](https://github.com/realm/realm-core/pull/7882))
+* Updated the return type of `LogCategory::get_category_names()` from `std::vector<const char*>` to `std::vector<std::string_view>`. ([PR #7879](https://github.com/realm/realm-core/pull/7879))
 * Added `realm_get_persisted_schema_version` for reading the version of the schema currently stored locally. (PR [#7873](https://github.com/realm/realm-core/pull/7873))
 
+### Fixed
+* <How do the end-user experience this issue? what was the impact?> ([#????](https://github.com/realm/realm-core/issues/????), since v?.?.?)
+* None.
+
+### Breaking changes
+* None.
+
+### Compatibility
+* Fileformat: Generates files with format v24. Reads and automatically upgrade from fileformat v10. If you want to upgrade from an earlier file format version you will have to use RealmCore v13.x.y or earlier.
+
+-----------
+
+### Internals
+* None.
+
+----------------------------------------------
+
+# 14.10.4 Release notes
+
+### Enhancements
+
 ### Fixed
 * When a public name is defined on a property, calling `realm::Results::sort()` or `realm::Results::distinct()` with the internal name could throw an error like `Cannot sort on key path 'NAME': property 'PersonObject.NAME' does not exist`. ([realm/realm-js#6779](https://github.com/realm/realm-js/issues/6779), since v12.12.0)
 
@@ -23,14 +45,14 @@
 # 14.10.3 Release notes
 
 ### Enhancements
-* "Next launch" metadata file actions are now performed in a multi-process safe manner ([#7576](https://github.com/realm/realm-core/pull/7576)).
+* "Next launch" metadata file actions are now performed in a multi-process safe manner. ([PR #7576](https://github.com/realm/realm-core/pull/7576))
 
 ### Fixed
 * Fixed a change of mode from Strong to All when removing links from an embedded object that links to a tombstone. This affects sync apps that use embedded objects which have a `Lst<Mixed>` that contains a link to another top level object which has been deleted by another sync client (creating a tombstone locally). In this particular case, the switch would cause any remaining link removals to recursively delete the destination object if there were no other links to it. ([#7828](https://github.com/realm/realm-core/issues/7828), since 14.0.0-beta.0)
 * Fixed removing backlinks from the wrong objects if the link came from a nested list, nested dictionary, top-level dictionary, or list of mixed, and the source table had more than 256 objects. This could manifest as `array_backlink.cpp:112: Assertion failed: int64_t(value >> 1) == key.value` when removing an object. ([#7594](https://github.com/realm/realm-core/issues/7594), since v11 for dictionaries)
 * Fixed the collapse/rejoin of clusters which contained nested collections with links. This could manifest as `array.cpp:319: Array::move() Assertion failed: begin <= end [2, 1]` when removing an object. ([#7839](https://github.com/realm/realm-core/issues/7839), since the introduction of nested collections in v14.0.0-beta.0)
 * wait_for_upload_completion() was inconsistent in how it handled commits which did not produce any changesets to upload. Previously it would sometimes complete immediately if all commits waiting to be uploaded were empty, and at other times it would wait for a server roundtrip. It will now always complete immediately. ([PR #7796](https://github.com/realm/realm-core/pull/7796)).
-* `realm_sync_session_handle_error_for_testing` parameter `is_fatal` was flipped changing the expected behavior. (#[7750](https://github.com/realm/realm-core/issues/7750)).
+* `realm_sync_session_handle_error_for_testing` parameter `is_fatal` was flipped changing the expected behavior. ([#7750](https://github.com/realm/realm-core/issues/7750))
 
 ### Breaking changes
 * None.

Package.swift

@@ -3,7 +3,7 @@
 import PackageDescription
 import Foundation
 
-let versionStr = "14.10.3"
+let versionStr = "14.10.4"
 let versionPieces = versionStr.split(separator: "-")
 let versionCompontents = versionPieces[0].split(separator: ".")
 let versionExtra = versionPieces.count > 1 ? versionPieces[1] : ""

dependencies.yml

@@ -1,5 +1,5 @@
 PACKAGE_NAME: realm-core
-VERSION: 14.10.3
+VERSION: 14.10.4
 OPENSSL_VERSION: 3.2.0
 ZLIB_VERSION: 1.2.13
 # https://github.com/10gen/baas/commits

src/realm/object-store/c_api/logging.cpp

@@ -91,7 +91,7 @@ RLM_API size_t realm_get_category_names(size_t num_values, const char** out_valu
         if (number_to_copy > num_values)
             number_to_copy = num_values;
         for (size_t n = 0; n < number_to_copy; n++) {
-            out_values[n] = vec[n];
+            out_values[n] = vec[n].data();
         }
     }
     return number_to_copy;

src/realm/object-store/c_api/schema.cpp

@@ -68,7 +68,7 @@ RLM_API uint64_t realm_get_persisted_schema_version(const realm_config_t* config
     }
 
     auto realm = Realm::get_shared_realm(conf);
-    uint64_t version = ObjectStore::get_schema_version(realm->get()->read_group());
+    uint64_t version = ObjectStore::get_schema_version(realm->read_group());
     return version;
 }
 

src/realm/sync/CMakeLists.txt

@@ -108,13 +108,11 @@ elseif(REALM_HAVE_OPENSSL)
     target_link_libraries(Sync PUBLIC OpenSSL::SSL)
 endif()
 
-if(WIN32 AND NOT WINDOWS_STORE)
-    target_link_libraries(Sync INTERFACE Version.lib)
-    if(CMAKE_VERSION VERSION_LESS "3.21")
-        # This is needed for OpenSSL, but CMake's FindOpenSSL didn't declare it
-        # on the OpenSSL::Crypto target until CMake 3.21.0. 
-        target_link_libraries(Sync INTERFACE Crypt32.lib)
+if(WIN32)
+    if(NOT WINDOWS_STORE)
+        target_link_libraries(Sync INTERFACE Version.lib)
     endif()
+    target_link_libraries(Sync INTERFACE Crypt32.lib)
 endif()
 
 install(TARGETS Sync EXPORT realm

src/realm/sync/network/network_ssl.cpp

@@ -7,13 +7,15 @@
 #include <realm/sync/network/network_ssl.hpp>
 
 #if REALM_HAVE_OPENSSL
+#include <openssl/conf.h>
+#include <openssl/x509v3.h>
 #ifdef _WIN32
+using osslX509_NAME = X509_NAME; // alias this before including wincrypt.h because it gets clobbered
 #include <Windows.h>
+#include <wincrypt.h>
 #else
 #include <pthread.h>
 #endif
-#include <openssl/conf.h>
-#include <openssl/x509v3.h>
 #elif REALM_HAVE_SECURE_TRANSPORT
 #include <fstream>
 #include <vector>
@@ -64,6 +66,122 @@ void populate_cert_store_with_included_certs(X509_STORE* store, std::error_code&
 
 #endif // REALM_INCLUDE_CERTS
 
+#if REALM_HAVE_OPENSSL && _WIN32
+
+/// Allow OpenSSL to look up certificates in the Windows Trusted Root Certification Authority list by implementing the
+/// X509_LOOKUP interface.
+class CapiLookup {
+public:
+    CapiLookup()
+    {
+        // Try to open the store in all of these locations sequentially. Many of them might not exist, and the
+        // CERT_STORE_OPEN_EXISTING_FLAG flag
+        // will cause CertOpenStore to return null in which case we just move on to the next.
+        // The order is important - we go from the most likely to the least likely to optimize lookup.
+        static std::initializer_list<DWORD> store_locations{CERT_SYSTEM_STORE_CURRENT_USER,
+                                                            CERT_SYSTEM_STORE_LOCAL_MACHINE,
+                                                            CERT_SYSTEM_STORE_CURRENT_SERVICE,
+                                                            CERT_SYSTEM_STORE_SERVICES,
+                                                            CERT_SYSTEM_STORE_USERS,
+                                                            CERT_SYSTEM_STORE_CURRENT_USER_GROUP_POLICY,
+                                                            CERT_SYSTEM_STORE_LOCAL_MACHINE_GROUP_POLICY,
+                                                            CERT_SYSTEM_STORE_LOCAL_MACHINE_ENTERPRISE};
+
+        for (DWORD location : store_locations) {
+            constexpr DWORD flags =
+                CERT_STORE_READONLY_FLAG | CERT_STORE_SHARE_CONTEXT_FLAG | CERT_STORE_OPEN_EXISTING_FLAG;
+            HCERTSTORE store = CertOpenStore(CERT_STORE_PROV_SYSTEM_W, 0, NULL, flags | location, L"ROOT");
+            if (store)
+                m_stores.push_back(store);
+        }
+    }
+
+    ~CapiLookup()
+    {
+        for (auto store : m_stores) {
+            CertCloseStore(store, 0);
+        }
+    }
+
+    int get_by_subject(X509_LOOKUP* ctx, X509_LOOKUP_TYPE type, const osslX509_NAME* name, X509_OBJECT* ret)
+    {
+        if (type != X509_LU_X509)
+            return 0;
+
+        // Convert the OpenSSL X509_NAME structure into its ASN.1 representation and construct a CAPI search parameter
+        CERT_NAME_BLOB capi_name = {0};
+        capi_name.cbData = i2d_X509_NAME(name, &capi_name.pbData);
+        int result = 0;
+
+        for (auto store : m_stores) {
+            PCCERT_CONTEXT cert =
+                CertFindCertificateInStore(store, X509_ASN_ENCODING, 0, CERT_FIND_SUBJECT_NAME, &capi_name, NULL);
+            if (!cert)
+                continue;
+
+            // Convert the ASN.1 representation of the CAPI certificate into an OpenSSL certificate and add it to the
+            // OpenSSL store
+            const unsigned char* encoded_cert_data = cert->pbCertEncoded;
+            X509* ossl_cert = d2i_X509(NULL, &encoded_cert_data, cert->cbCertEncoded);
+            result = X509_STORE_add_cert(X509_LOOKUP_get_store(ctx), ossl_cert);
+            X509_free(ossl_cert);
+            break;
+        }
+
+        OPENSSL_free(capi_name.pbData);
+
+        // if we previously added a certificate to the store we need to look it up again from the store and return
+        // that
+        if (result)
+            result = get_cached_object(ctx, type, name, ret);
+        return result;
+    }
+
+private:
+    int get_cached_object(X509_LOOKUP* ctx, X509_LOOKUP_TYPE type, const osslX509_NAME* name, X509_OBJECT* ret)
+    {
+        REALM_ASSERT_RELEASE(type == X509_LU_X509);
+
+        // Loop through the objects already in the store to find the one we just added in get_by_subject.
+        // retrieve_by_subject returns a cert with refcount 1 but set1_X509 increases it.
+        // That's why we need to free it after before returning, otherwise it will leak.
+
+        STACK_OF(X509_OBJECT)* objects = X509_STORE_get0_objects(X509_LOOKUP_get_store(ctx));
+        X509_OBJECT* tmp = X509_OBJECT_retrieve_by_subject(objects, type, name);
+        if (!tmp)
+            return 0;
+
+        X509* cert = X509_OBJECT_get0_X509(tmp);
+        int result = X509_OBJECT_set1_X509(ret, cert);
+        X509_free(cert);
+
+        return result;
+    }
+
+    std::vector<HCERTSTORE> m_stores;
+};
+
+void add_windows_certificate_store_lookup(X509_STORE* store)
+{
+    X509_LOOKUP_METHOD* capi_lookup = X509_LOOKUP_meth_new("capi");
+
+    X509_LOOKUP_meth_set_new_item(capi_lookup, [](X509_LOOKUP* ctx) {
+        auto* data = new CapiLookup();
+        return X509_LOOKUP_set_method_data(ctx, data);
+    });
+    X509_LOOKUP_meth_set_free(capi_lookup, [](X509_LOOKUP* ctx) {
+        auto* data = reinterpret_cast<CapiLookup*>(X509_LOOKUP_get_method_data(ctx));
+        delete data;
+    });
+    X509_LOOKUP_meth_set_get_by_subject(capi_lookup, [](auto ctx, auto type, auto name, auto ret) {
+        auto* data = reinterpret_cast<CapiLookup*>(X509_LOOKUP_get_method_data(ctx));
+        return data->get_by_subject(ctx, type, name, ret);
+    });
+
+    X509_STORE_add_lookup(store, capi_lookup);
+}
+
+#endif // REALM_HAVE_OPENSSL && _WIN32
 
 #if REALM_HAVE_OPENSSL && (OPENSSL_VERSION_NUMBER < 0x10100000L || defined(LIBRESSL_VERSION_NUMBER))
 
@@ -330,6 +448,9 @@ void Context::ssl_use_default_verify(std::error_code& ec)
         ec = std::error_code(int(ERR_get_error()), openssl_error_category);
         return;
     }
+#endif
+#ifdef _WIN32
+    add_windows_certificate_store_lookup(SSL_CTX_get_cert_store(m_ssl_ctx));
 #endif
     ec = std::error_code();
 }

src/realm/sync/network/network_ssl.hpp

@@ -131,6 +131,8 @@ class Context {
     /// default certificates for server verification. For OpenSSL,
     /// use_default_verify() corresponds to
     /// SSL_CTX_set_default_verify_paths(SSL_CTX*);
+    ///
+    /// On Windows this also adds a lookup to the system Trusted Root Certification Authorities list.
     void use_default_verify();
 
     /// The verify file is a PEM file containing trust certificates that the

src/realm/util/logger.cpp

@@ -30,7 +30,7 @@ std::shared_ptr<util::Logger> s_default_logger;
 } // anonymous namespace
 
 size_t LogCategory::s_next_index = 0;
-static std::map<std::string_view, LogCategory*> log_catagory_map;
+static std::map<std::string_view, LogCategory*> log_category_map;
 
 LogCategory LogCategory::realm("Realm", nullptr);
 LogCategory LogCategory::storage("Storage", &realm);
@@ -58,19 +58,20 @@ LogCategory::LogCategory(std::string_view name, LogCategory* parent)
         parent->m_children.push_back(this);
     }
     m_name += name;
-    log_catagory_map.emplace(m_name, this);
+    log_category_map.emplace(m_name, this);
 }
 
 LogCategory& LogCategory::get_category(std::string_view name)
 {
-    return *log_catagory_map.at(name); // Throws
+    return *log_category_map.at(name); // Throws
 }
 
-std::vector<const char*> LogCategory::get_category_names()
+std::vector<std::string_view> LogCategory::get_category_names()
 {
-    std::vector<const char*> ret;
-    for (auto& it : log_catagory_map) {
-        ret.push_back(it.second->get_name().c_str());
+    std::vector<std::string_view> ret;
+    ret.reserve(log_category_map.size());
+    for (auto& it : log_category_map) {
+        ret.push_back(it.second->get_name());
     }
     return ret;
 }

src/realm/util/logger.hpp

@@ -99,7 +99,7 @@ class LogCategory {
     // Find category from fully qualified name. Will throw if
     // name does not match a category
     static LogCategory& get_category(std::string_view name);
-    static std::vector<const char*> get_category_names();
+    static std::vector<std::string_view> get_category_names();
 
 private:
     friend class Logger;

test/test_util_network_ssl.cpp

@@ -6,6 +6,10 @@
 #include "test.hpp"
 #include "util/semaphore.hpp"
 
+#ifdef _WIN32
+#include <wincrypt.h>
+#endif
+
 using namespace realm;
 using namespace realm::sync;
 using namespace realm::test_util;
@@ -1216,4 +1220,73 @@ TEST(Util_Network_SSL_Certificate_Failure)
     thread_2.join();
 }
 
+#ifdef _WIN32
+
+// Adding a trusted root certificate causes a blocking popup to appear so only run this test
+// if in an interactive session with a debugger attached, otherwise CI machines will block.
+// TODO: maybe use the CI environment variable for the condition?
+TEST_IF(Util_Network_SSL_Certificate_From_Windows_Cert_Store, IsDebuggerPresent())
+{
+    std::string ca_dir = get_test_resource_path();
+
+    BIO* file = BIO_new_file((ca_dir + "crt.pem").c_str(), "rb");
+    X509* cert = PEM_read_bio_X509(file, NULL, 0, NULL);
+    BIO_free(file);
+
+    BIO* mem = BIO_new(BIO_s_mem());
+    int size = i2d_X509_bio(mem, cert);
+    BUF_MEM* buffer;
+    BIO_get_mem_ptr(mem, &buffer);
+
+    PCCERT_CONTEXT cert_context;
+    HCERTSTORE store = CertOpenSystemStoreA(NULL, "ROOT");
+    CertAddEncodedCertificateToStore(store, X509_ASN_ENCODING, reinterpret_cast<const BYTE*>(buffer->data),
+                                     static_cast<DWORD>(buffer->length), CERT_STORE_ADD_USE_EXISTING, &cert_context);
+    BIO_free(mem);
+
+    network::Service service_1, service_2;
+    network::Socket socket_1{service_1}, socket_2{service_2};
+    network::ssl::Context ssl_context_1;
+    network::ssl::Context ssl_context_2;
+
+    ssl_context_1.use_certificate_chain_file(ca_dir + "dns-chain.crt.pem");
+    ssl_context_1.use_private_key_file(ca_dir + "dns-checked-server.key.pem");
+    ssl_context_2
+        .use_default_verify(); // this will import the Windows certificates in the context's certificate store
+
+    network::ssl::Stream ssl_stream_1{socket_1, ssl_context_1, network::ssl::Stream::server};
+    network::ssl::Stream ssl_stream_2{socket_2, ssl_context_2, network::ssl::Stream::client};
+    ssl_stream_1.set_logger(test_context.logger.get());
+    ssl_stream_2.set_logger(test_context.logger.get());
+
+    ssl_stream_2.set_verify_mode(network::ssl::VerifyMode::peer);
+
+    // We expect success because the certificate is signed for www.example.com
+    // in both Common Name and SAN.
+    ssl_stream_2.set_host_name("www.example.com");
+
+    connect_sockets(socket_1, socket_2);
+
+    auto connector = [&] {
+        std::error_code ec;
+        ssl_stream_2.handshake(ec);
+        CHECK_EQUAL(std::error_code(), ec);
+    };
+    auto acceptor = [&] {
+        std::error_code ec;
+        ssl_stream_1.handshake(ec);
+        CHECK_EQUAL(std::error_code(), ec);
+    };
+
+    std::thread thread_1(std::move(connector));
+    std::thread thread_2(std::move(acceptor));
+    thread_1.join();
+    thread_2.join();
+
+    CertDeleteCertificateFromStore(cert_context);
+    CertCloseStore(store, 0);
+}
+
+#endif // _WIN32
+
 #endif // !REALM_MOBILE